chn/nixpkgs
1
0
Fork 0
mirror of https://github.com/CHN-beta/nixpkgs.git synced 2026-01-12 02:40:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Backport release-25.05] workflows/eval: pass outpaths via cachix instead of artifacts (#435863)

Browse Source
This commit is contained in:
Wolfgang Walther
2025-08-22 12:12:54 +00:00
committed by GitHub
parent 88c238e6b2 1646453f34
commit a6fede312b
6 changed files with 63 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.github/actions/checkout/action.yml vendored
View File

@@ -5,8 +5,6 @@ description: 'Checkout into trusted / untrusted / pinned folders consistently.'
inputs:
merged-as-untrusted-at:
description: "Whether and which SHA to checkout for the merge commit in the ./untrusted folder."
pinned-from:
description: "Whether to checkout the pinned nixpkgs for CI and from where (trusted, untrusted)."
target-as-trusted-at:
description: "Whether and which SHA to checkout for the target commit in the ./trusted folder."
@@ -16,7 +14,6 @@ runs:
- uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
env:
MERGED_SHA: ${{ inputs.merged-as-untrusted-at }}
PINNED_FROM: ${{ inputs.pinned-from }}
TARGET_SHA: ${{ inputs.target-as-trusted-at }}
with:
script: |
@@ -45,6 +42,7 @@ runs:
// A single fetch call comes with a lot less overhead. The fetch takes essentially the
// same time no matter whether its 1, 2 or 3 commits at once.
async function getPinnedSha(ref) {
if (!ref) return undefined
const { content, encoding } = (await github.rest.repos.getContent({
...context.repo,
path: 'ci/pinned.json',
@@ -60,16 +58,16 @@ runs:
path: 'untrusted',
},
{
sha: process.env.PINNED_FROM === 'untrusted' && (await getPinnedSha(process.env.MERGED_SHA)),
path: 'pinned'
sha: await getPinnedSha(process.env.MERGED_SHA),
path: 'untrusted-pinned'
},
{
sha: process.env.TARGET_SHA,
path: 'trusted',
},
{
sha: process.env.PINNED_FROM === 'trusted' && (await getPinnedSha(process.env.TARGET_SHA)),
path: 'pinned'
sha: await getPinnedSha(process.env.TARGET_SHA),
path: 'trusted-pinned'
}
].filter(({ sha }) => Boolean(sha))

15
.github/workflows/build.yml vendored
View File

@@ -51,7 +51,6 @@ jobs:
uses: ./.github/actions/checkout
with:
merged-as-untrusted-at: ${{ inputs.mergedSha }}
pinned-from: untrusted
- uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
with:
@@ -66,33 +65,33 @@ jobs:
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
pushFilter: '(-source$|-nixpkgs-tarball-)'
- run: nix-env --install -f nixpkgs/pinned -A nix-build-uncached
- run: nix-env --install -f nixpkgs/untrusted-pinned -A nix-build-uncached
- name: Build shell
if: contains(matrix.builds, 'shell')
run: echo "${{ matrix.systems }}" | xargs -n1 nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A shell --argstr system
run: echo "${{ matrix.systems }}" | xargs -n1 nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A shell --argstr system
- name: Build NixOS manual
if: |
contains(matrix.builds, 'manual-nixos') && !cancelled() &&
contains(fromJSON(inputs.baseBranch).type, 'primary')
run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A manual-nixos --out-link nixos-manual
run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A manual-nixos --out-link nixos-manual
- name: Build Nixpkgs manual
if: contains(matrix.builds, 'manual-nixpkgs') && !cancelled()
run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A manual-nixpkgs -A manual-nixpkgs-tests
run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A manual-nixpkgs -A manual-nixpkgs-tests
- name: Build Nixpkgs manual tests
if: contains(matrix.builds, 'manual-nixpkgs-tests') && !cancelled()
run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A manual-nixpkgs-tests
run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A manual-nixpkgs-tests
- name: Build lib tests
if: contains(matrix.builds, 'lib-tests') && !cancelled()
run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A lib-tests
run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A lib-tests
- name: Build tarball
if: contains(matrix.builds, 'tarball') && !cancelled()
run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A tarball
run: nix-build-uncached nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A tarball
- name: Upload NixOS manual
if: |

3
.github/workflows/check.yml vendored
View File

@@ -103,7 +103,6 @@ jobs:
uses: ./.github/actions/checkout
with:
merged-as-untrusted-at: ${{ inputs.mergedSha }}
pinned-from: trusted
target-as-trusted-at: ${{ inputs.targetSha }}
- uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
@@ -117,7 +116,7 @@ jobs:
pushFilter: -source$
- name: Build codeowners validator
run: nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/pinned -A codeownersValidator
run: nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A codeownersValidator
- uses: actions/create-github-app-token@0f859bf9e69e887678d5bbfbee594437cb440ffe # v2.1.0
if: github.event_name == 'pull_request_target' && vars.OWNER_RO_APP_ID

127
.github/workflows/eval.yml vendored
View File

@@ -71,8 +71,6 @@ jobs:
# to not interrupt main Eval's compare step.
continue-on-error: ${{ matrix.version != '' }}
name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }}
outputs:
targetRunId: ${{ steps.targetRunId.outputs.targetRunId }}
timeout-minutes: 15
steps:
# This is not supposed to be used and just acts as a fallback.
@@ -89,11 +87,11 @@ jobs:
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
with:
sparse-checkout: .github/actions
- name: Check out the PR at the test merge commit
- name: Check out the PR at merged and target commits
uses: ./.github/actions/checkout
with:
merged-as-untrusted-at: ${{ inputs.mergedSha }}
pinned-from: untrusted
target-as-trusted-at: ${{ inputs.targetSha }}
- name: Install Nix
uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
@@ -106,98 +104,64 @@ jobs:
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
pushFilter: '(-source|-single-chunk)$'
- name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes
- name: Evaluate the ${{ matrix.system }} output paths at the merge commit
env:
MATRIX_SYSTEM: ${{ matrix.system }}
MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }}
run: |
nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A eval.singleSystem \
nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.singleSystem \
--argstr evalSystem "$MATRIX_SYSTEM" \
--arg chunkSize 8000 \
--argstr nixPath "$MATRIX_VERSION" \
--out-link merged
# If it uses too much memory, slightly decrease chunkSize
# If it uses too much memory, slightly decrease chunkSize.
# Note: Keep the same further down in sync!
- name: Upload the output paths and eval stats
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}merged-${{ matrix.system }}
path: merged/*
- name: Log current API rate limits
env:
GH_TOKEN: ${{ github.token }}
run: gh api /rate_limit | jq
- name: Get target run id
# Running the attrpath generation step separately from the outpath step afterwards.
# The idea is that, *if* Eval on the target branch has not finished, yet, we will
# generate the attrpaths in the meantime - and the separate command command afterwards
# will check cachix again for whether Eval has finished. If no Eval result from the
# target branch can be found the second time, we proceed to run it in here. Attrpaths
# generation takes roughly 30 seconds, so for every normal use-case this should be more
# than enough of a head start for Eval on the target branch to finish.
# This edge-case, that Eval on the target branch is delayed is unlikely to happen anyway:
# For a commit to become the target commit of a PR, it must *already* be on the branch.
# Normally, CI should always start running on that push event *before* it starts running
# on the PR.
- name: Evaluate the ${{ matrix.system }} attribute paths at the target commit
if: inputs.targetSha
id: targetRunId
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
env:
MATRIX_SYSTEM: ${{ matrix.system }}
TARGET_SHA: ${{ inputs.targetSha }}
with:
script: |
const system = process.env.MATRIX_SYSTEM
const targetSha = process.env.TARGET_SHA
let run_id
try {
run_id = (await github.rest.actions.listWorkflowRuns({
...context.repo,
workflow_id: 'push.yml',
event: 'push',
head_sha: targetSha
})).data.workflow_runs[0].id
} catch {
throw new Error(`Could not find a push.yml workflow run for ${targetSha}.`)
}
// Waiting 120 * 5 sec = 10 min. max.
// Eval takes max 5-6 minutes, normally.
for (let i = 0; i < 120; i++) {
const result = await github.rest.actions.listWorkflowRunArtifacts({
...context.repo,
run_id,
name: `merged-${system}`
})
if (result.data.total_count > 0) {
core.setOutput('targetRunId', run_id)
return
}
await new Promise(resolve => setTimeout(resolve, 5000))
}
// No artifact found at this stage. This usually means that Eval failed on the target branch.
// This should only happen when Eval is broken on the target branch and this PR fixes it.
// Continue without targetRunId to skip the remaining steps, but pass the job.
- name: Log current API rate limits
env:
GH_TOKEN: ${{ github.token }}
run: gh api /rate_limit | jq
- uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
if: steps.targetRunId.outputs.targetRunId
with:
run-id: ${{ steps.targetRunId.outputs.targetRunId }}
name: merged-${{ matrix.system }}
path: target
github-token: ${{ github.token }}
merge-multiple: true
- name: Compare outpaths against the target branch
if: steps.targetRunId.outputs.targetRunId
env:
MATRIX_SYSTEM: ${{ matrix.system }}
run: |
nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A eval.diff \
--arg beforeDir ./target \
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.attrpathsSuperset \
--argstr evalSystem "$MATRIX_SYSTEM" \
--argstr nixPath "nixVersions.latest"
- name: Evaluate the ${{ matrix.system }} output paths at the target commit
if: inputs.targetSha
env:
MATRIX_SYSTEM: ${{ matrix.system }}
# This should be very quick, because it pulls the eval results from Cachix.
run: |
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.singleSystem \
--argstr evalSystem "$MATRIX_SYSTEM" \
--arg chunkSize 8000 \
--argstr nixPath "nixVersions.latest" \
--out-link target
- name: Compare outpaths against the target branch
if: inputs.targetSha
env:
MATRIX_SYSTEM: ${{ matrix.system }}
run: |
nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.diff \
--arg beforeDir "$(readlink ./target)" \
--arg afterDir "$(readlink ./merged)" \
--argstr evalSystem "$MATRIX_SYSTEM" \
--out-link diff
- name: Upload outpaths diff and stats
if: steps.targetRunId.outputs.targetRunId
if: inputs.targetSha
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }}
@@ -206,7 +170,7 @@ jobs:
compare:
runs-on: ubuntu-24.04-arm
needs: [eval]
if: needs.eval.outputs.targetRunId && !cancelled() && !failure()
if: inputs.targetSha && !cancelled() && !failure()
permissions:
statuses: write
timeout-minutes: 5
@@ -219,7 +183,6 @@ jobs:
with:
merged-as-untrusted-at: ${{ inputs.mergedSha }}
target-as-trusted-at: ${{ inputs.targetSha }}
pinned-from: trusted
- name: Download output paths and eval stats for all systems
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
@@ -233,7 +196,7 @@ jobs:
- name: Combine all output paths and eval stats
run: |
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/pinned -A eval.combine \
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.combine \
--arg diffDir ./diff \
--out-link combined
@@ -245,7 +208,7 @@ jobs:
| jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json
# Use the target branch to get accurate maintainer info
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/pinned -A eval.compare \
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.compare \
--arg combinedDir "$(realpath ./combined)" \
--arg touchedFilesJson ./touched-files.json \
--argstr githubAuthorId "$AUTHOR_ID" \

9
.github/workflows/lint.yml vendored
View File

@@ -31,7 +31,6 @@ jobs:
uses: ./.github/actions/checkout
with:
merged-as-untrusted-at: ${{ inputs.mergedSha }}
pinned-from: untrusted
- uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
@@ -44,7 +43,7 @@ jobs:
# Note that it's fine to run this on untrusted code because:
# - There's no secrets accessible here
# - The build is sandboxed
if ! nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A fmt.check; then
if ! nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A fmt.check; then
echo "Some files are not properly formatted"
echo "Please format them by going to the Nixpkgs root directory and running one of:"
echo " nix-shell --run treefmt"
@@ -66,7 +65,6 @@ jobs:
uses: ./.github/actions/checkout
with:
merged-as-untrusted-at: ${{ inputs.mergedSha }}
pinned-from: untrusted
- uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
@@ -81,7 +79,7 @@ jobs:
- name: Parse all nix files
run: |
# Tests multiple versions at once, let's make sure all of them run, so keep-going.
nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A parse --keep-going
nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A parse --keep-going
nixpkgs-vet:
runs-on: ubuntu-24.04-arm
@@ -94,7 +92,6 @@ jobs:
uses: ./.github/actions/checkout
with:
merged-as-untrusted-at: ${{ inputs.mergedSha }}
pinned-from: untrusted
target-as-trusted-at: ${{ inputs.targetSha }}
- uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31
@@ -112,7 +109,7 @@ jobs:
# Force terminal colors to be enabled. The library that `nixpkgs-vet` uses respects https://bixense.com/clicolors/
CLICOLOR_FORCE: 1
run: |
if nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/pinned -A nixpkgs-vet --arg base "./nixpkgs/trusted" --arg head "./nixpkgs/untrusted"; then
if nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A nixpkgs-vet --arg base "./nixpkgs/trusted" --arg head "./nixpkgs/untrusted"; then
exit 0
else
exitCode=$?

2
ci/eval/default.nix
View File

@@ -141,6 +141,8 @@ let
env = {
inherit evalSystem chunkSize;
};
__structuredAttrs = true;
unsafeDiscardReferences.out = true;
}
''
export NIX_STATE_DIR=$(mktemp -d)