nativelink/.github/workflows/main.yaml at main · TraceMachina/nativelink · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
name: CI

# Controls when the workflow will run.
on:
  push:
    branches: [ main ]
    paths-ignore:
      - '.github/styles/**'
      - 'web/**'
  pull_request:
    branches: [ main ]
    paths-ignore:
      - '.github/styles/**'
      - 'web/**'

permissions: read-all

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
  # TODO(palfrey): Flaky. Fix.
  # docker-compose-compiles-nativelink:
  #   # The type of runner that the job will run on.
  #   runs-on: ubuntu-24.04
  #   strategy:
  #     matrix:
  #       # Which OS versions we will test on.
  #       os_version: [ 24.04 ]
  #   steps:
  #   - name: Checkout
  #     uses: >- # v6.0.2
  #       actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd

  #   - name: Set up Docker Buildx
  #     uses: >- # v3.9.0
  #       docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca

  #   - name: Build Nativelink image
  #     uses: >- # v6.13.0
  #       docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991
  #     with:
  #       context: .
  #       file: ./deployment-examples/docker-compose/Dockerfile
  #       build-args: |
  #         OPT_LEVEL=opt
  #         OS_VERSION=${{ matrix.os_version }}
  #         ADDITIONAL_SETUP_WORKER_CMD=apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y gcc g++ lld pkg-config python3
  #       load: true # This brings the build into `docker images` from buildx.
  #       tags: trace_machina/nativelink:latest

  #   - name: Build builder image
  #     uses: >- # v6.13.0
  #       docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991
  #     with:
  #       context: .
  #       file: ./deployment-examples/docker-compose/Dockerfile
  #       build-args: |
  #         OPT_LEVEL=opt
  #         OS_VERSION=${{ matrix.os_version }}
  #       load: true # This brings the build into `docker images` from buildx.
  #       tags: trace_machina/nativelink:builder
  #       target: builder

  #   - name: Compile NativeLink with NativeLink
  #     run: |
  #       mkdir -p ~/.cache && \
  #       cd deployment-examples/docker-compose && \
  #       (docker-compose up -d || docker compose up -d) && \
  #       cd ../../ && \
  #       docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
  #         bazel clean && \
  #         bazel test //... \
  #         --extra_toolchains=@rust_toolchains//:all \
  #         --remote_cache=grpc://127.0.0.1:50051 \
  #         --remote_executor=grpc://127.0.0.1:50052 \
  #         --remote_default_exec_properties=cpu_count=2 \
  #       ' && \
  #       docker run --rm --net=host -w /root/nativelink -v $PWD:/root/nativelink trace_machina/nativelink:builder sh -c ' \
  #         bazel clean && \
  #         bazel test //... \
  #         --extra_toolchains=@rust_toolchains//:all \
  #         --remote_cache=grpc://127.0.0.1:50051 \
  #         --remote_executor=grpc://127.0.0.1:50052 \
  #         --remote_default_exec_properties=cpu_count=2 \
  #       ' 2>&1 | ( ! grep '         PASSED in ' ) # If we get PASSED without (cache) it means there's a cache issue.

  integration-tests:
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    steps:
    - name: Checkout
      uses: >- # v6.0.2
        actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd

    - name: Setup NativeLink Cloud
      uses: ./.github/actions/setup-nativelink-cloud
      with:
        ci-mode: ${{ vars.NATIVELINK_CI_MODE }}
        claim-base: ${{ secrets.NATIVELINK_STAGING_CLAIM_BASE }}
        bes-results-url: ${{ secrets.NATIVELINK_STAGING_BES_RESULTS_URL }}
        api-key: >-
          ${{ github.event_name == 'push'
              && secrets.NATIVELINK_STAGING_API_KEY_RW
              || secrets.NATIVELINK_STAGING_API_KEY_RO }}
        mode: ${{ github.event_name == 'push' && 'write' || 'read' }}
        exec: ${{ runner.os == 'Linux' && 'on' || 'off' }}

    - name: Set up Docker Buildx
      uses: >- # v3.9.0
        docker/setup-buildx-action@f7ce87c1d6bead3e36075b2ce75da1f6cc28aaca

    - name: Build image
      uses: >- # v6.13.0
        docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991
      with:
        context: .
        file: ./deployment-examples/docker-compose/Dockerfile
        build-args: |
          OPT_LEVEL=fastbuild
          ADDITIONAL_BAZEL_FLAGS=--extra_toolchains=@rust_toolchains//:all
        secret-files: |
          nativelink-bazelrc=user.bazelrc
        load: true # This brings the build into `docker images` from buildx.
        tags: trace_machina/nativelink:latest

    - name: Run tests
      env:
        # Merges to main are a trusted upload lane; PRs are not (keeps the
        # repo-wide --remote_upload_local_results=false anti-poisoning default
        # in force for PRs). Upload-dependent integration tests
        # (chunking_cache_test, simple_cache_test) self-skip unless this is 1.
        NL_LOCAL_UPLOADS: ${{ github.event_name == 'push' && '1' || '0' }}
      run: ./run_integration_tests.sh