From 1646453f34812059aa625e0ddc1e0b8ff5b0fe1a Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 19 Aug 2025 17:47:27 +0200 Subject: [PATCH] workflows/eval: pass outpaths via cachix instead of artifacts Instead of uploading the outpaths as artifact, this uploads them via cachix. Most of all, this makes CI less brittle, because Eval in PRs will still be able to succeed, even if no workflow run for the push event could be found on the target branch. It will just take longer. This also makes moving Eval into the Merge Queue easier to do: When downloading artifacts from a different run, these would always have to match on the right event, too. By pulling from cachix, the same workflow can support target branches with merge queue and without merge queue at the same time. The latter would still use the push event, while the former could use the merge_group event. Last but not least, this should fix Eval on PRs targeting `wip-` branches and any other branches that the push event doesn't trigger on. These would never find an Eval result from the target branch and could never show rebuilds accurately. Now these PRs should work at a slightly higher runtime cost. (cherry picked from commit c1b06db57b316c99aa1933778343de7f90266640) --- .github/workflows/eval.yml | 105 +++++++++++++------------------------ ci/eval/default.nix | 2 + 2 files changed, 37 insertions(+), 70 deletions(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index a2b16638b910..7efa71913ac7 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -71,8 +71,6 @@ jobs: # to not interrupt main Eval's compare step. continue-on-error: ${{ matrix.version != '' }} name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }} - outputs: - targetRunId: ${{ steps.targetRunId.outputs.targetRunId }} timeout-minutes: 15 steps: # This is not supposed to be used and just acts as a fallback. @@ -89,10 +87,11 @@ jobs: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: sparse-checkout: .github/actions - - name: Check out the PR at the test merge commit + - name: Check out the PR at merged and target commits uses: ./.github/actions/checkout with: merged-as-untrusted-at: ${{ inputs.mergedSha }} + target-as-trusted-at: ${{ inputs.targetSha }} - name: Install Nix uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31 @@ -105,7 +104,7 @@ jobs: authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} pushFilter: '(-source|-single-chunk)$' - - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes + - name: Evaluate the ${{ matrix.system }} output paths at the merge commit env: MATRIX_SYSTEM: ${{ matrix.system }} MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }} @@ -115,88 +114,54 @@ jobs: --arg chunkSize 8000 \ --argstr nixPath "$MATRIX_VERSION" \ --out-link merged - # If it uses too much memory, slightly decrease chunkSize + # If it uses too much memory, slightly decrease chunkSize. + # Note: Keep the same further down in sync! - - name: Upload the output paths and eval stats - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}merged-${{ matrix.system }} - path: merged/* - - - name: Log current API rate limits - env: - GH_TOKEN: ${{ github.token }} - run: gh api /rate_limit | jq - - - name: Get target run id + # Running the attrpath generation step separately from the outpath step afterwards. + # The idea is that, *if* Eval on the target branch has not finished, yet, we will + # generate the attrpaths in the meantime - and the separate command command afterwards + # will check cachix again for whether Eval has finished. If no Eval result from the + # target branch can be found the second time, we proceed to run it in here. Attrpaths + # generation takes roughly 30 seconds, so for every normal use-case this should be more + # than enough of a head start for Eval on the target branch to finish. + # This edge-case, that Eval on the target branch is delayed is unlikely to happen anyway: + # For a commit to become the target commit of a PR, it must *already* be on the branch. + # Normally, CI should always start running on that push event *before* it starts running + # on the PR. + - name: Evaluate the ${{ matrix.system }} attribute paths at the target commit if: inputs.targetSha - id: targetRunId - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 env: MATRIX_SYSTEM: ${{ matrix.system }} - TARGET_SHA: ${{ inputs.targetSha }} - with: - script: | - const system = process.env.MATRIX_SYSTEM - const targetSha = process.env.TARGET_SHA + run: | + nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.attrpathsSuperset \ + --argstr evalSystem "$MATRIX_SYSTEM" \ + --argstr nixPath "nixVersions.latest" - let run_id - try { - run_id = (await github.rest.actions.listWorkflowRuns({ - ...context.repo, - workflow_id: 'push.yml', - event: 'push', - head_sha: targetSha - })).data.workflow_runs[0].id - } catch { - throw new Error(`Could not find a push.yml workflow run for ${targetSha}.`) - } - - // Waiting 120 * 5 sec = 10 min. max. - // Eval takes max 5-6 minutes, normally. - for (let i = 0; i < 120; i++) { - const result = await github.rest.actions.listWorkflowRunArtifacts({ - ...context.repo, - run_id, - name: `merged-${system}` - }) - if (result.data.total_count > 0) { - core.setOutput('targetRunId', run_id) - return - } - await new Promise(resolve => setTimeout(resolve, 5000)) - } - // No artifact found at this stage. This usually means that Eval failed on the target branch. - // This should only happen when Eval is broken on the target branch and this PR fixes it. - // Continue without targetRunId to skip the remaining steps, but pass the job. - - - name: Log current API rate limits + - name: Evaluate the ${{ matrix.system }} output paths at the target commit + if: inputs.targetSha env: - GH_TOKEN: ${{ github.token }} - run: gh api /rate_limit | jq - - - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 - if: steps.targetRunId.outputs.targetRunId - with: - run-id: ${{ steps.targetRunId.outputs.targetRunId }} - name: merged-${{ matrix.system }} - path: target - github-token: ${{ github.token }} - merge-multiple: true + MATRIX_SYSTEM: ${{ matrix.system }} + # This should be very quick, because it pulls the eval results from Cachix. + run: | + nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.singleSystem \ + --argstr evalSystem "$MATRIX_SYSTEM" \ + --arg chunkSize 8000 \ + --argstr nixPath "nixVersions.latest" \ + --out-link target - name: Compare outpaths against the target branch - if: steps.targetRunId.outputs.targetRunId + if: inputs.targetSha env: MATRIX_SYSTEM: ${{ matrix.system }} run: | nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.diff \ - --arg beforeDir ./target \ + --arg beforeDir "$(readlink ./target)" \ --arg afterDir "$(readlink ./merged)" \ --argstr evalSystem "$MATRIX_SYSTEM" \ --out-link diff - name: Upload outpaths diff and stats - if: steps.targetRunId.outputs.targetRunId + if: inputs.targetSha uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }} @@ -205,7 +170,7 @@ jobs: compare: runs-on: ubuntu-24.04-arm needs: [eval] - if: needs.eval.outputs.targetRunId && !cancelled() && !failure() + if: inputs.targetSha && !cancelled() && !failure() permissions: statuses: write timeout-minutes: 5 diff --git a/ci/eval/default.nix b/ci/eval/default.nix index 363804cce716..8d79034db59e 100644 --- a/ci/eval/default.nix +++ b/ci/eval/default.nix @@ -141,6 +141,8 @@ let env = { inherit evalSystem chunkSize; }; + __structuredAttrs = true; + unsafeDiscardReferences.out = true; } '' export NIX_STATE_DIR=$(mktemp -d)