mirror of
https://github.com/CHN-beta/nixpkgs.git
synced 2026-01-12 02:40:31 +08:00
6c105942ca
Due to how we pass in existing store paths via CLI arguments for the
diff and combine scripts, Nix didn't register a dependency on the store
paths properly. This meant that some of the derivations that were built,
didn't have the right store paths made available in the sandbox -
leading to all kinds of "not found" errors.
We worked around this in CI by resolving the symlinks to the nix store
beforehand. We tried to work around this locally by storing the nix
store path in BASELINE, but this didn't fully work. By explicitly
registering these store paths as dependencies, this should work across
the board - without any magic required by the caller.
(cherry picked from commit 45a765a282)
362 lines
14 KiB
YAML
362 lines
14 KiB
YAML
name: Eval
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
mergedSha:
|
|
required: true
|
|
type: string
|
|
targetSha:
|
|
type: string
|
|
systems:
|
|
required: true
|
|
type: string
|
|
testVersions:
|
|
required: false
|
|
default: false
|
|
type: boolean
|
|
secrets:
|
|
CACHIX_AUTH_TOKEN:
|
|
required: true
|
|
|
|
permissions: {}
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash
|
|
|
|
jobs:
|
|
versions:
|
|
if: inputs.testVersions
|
|
runs-on: ubuntu-24.04-arm
|
|
outputs:
|
|
versions: ${{ steps.versions.outputs.versions }}
|
|
steps:
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
path: trusted
|
|
sparse-checkout: |
|
|
ci/supportedVersions.nix
|
|
|
|
- name: Check out the PR at the test merge commit
|
|
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
ref: ${{ inputs.mergedSha }}
|
|
path: untrusted
|
|
sparse-checkout: |
|
|
ci/pinned.json
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31
|
|
|
|
- name: Load supported versions
|
|
id: versions
|
|
run: |
|
|
echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT"
|
|
|
|
eval:
|
|
runs-on: ubuntu-24.04-arm
|
|
needs: versions
|
|
if: ${{ !cancelled() && !failure() }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
system: ${{ fromJSON(inputs.systems) }}
|
|
version:
|
|
- "" # Default Eval triggering rebuild labels and such.
|
|
- ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates.
|
|
# Failures for versioned Evals will be collected in a separate job below
|
|
# to not interrupt main Eval's compare step.
|
|
continue-on-error: ${{ matrix.version != '' }}
|
|
name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }}
|
|
timeout-minutes: 15
|
|
steps:
|
|
# This is not supposed to be used and just acts as a fallback.
|
|
# Without swap, when Eval runs OOM, it will fail badly with a
|
|
# job that is sometimes not interruptible anymore.
|
|
# If Eval starts swapping, decrease chunkSize to keep it fast.
|
|
- name: Enable swap
|
|
run: |
|
|
sudo fallocate -l 10G /swap
|
|
sudo chmod 600 /swap
|
|
sudo mkswap /swap
|
|
sudo swapon /swap
|
|
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
sparse-checkout: .github/actions
|
|
- name: Check out the PR at merged and target commits
|
|
uses: ./.github/actions/checkout
|
|
with:
|
|
merged-as-untrusted-at: ${{ inputs.mergedSha }}
|
|
target-as-trusted-at: ${{ inputs.targetSha }}
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31
|
|
|
|
- uses: cachix/cachix-action@0fc020193b5a1fa3ac4575aa3a7d3aa6a35435ad # v16
|
|
with:
|
|
# The nixpkgs-ci cache should not be trusted or used outside of Nixpkgs and its forks' CI.
|
|
name: ${{ vars.CACHIX_NAME || 'nixpkgs-ci' }}
|
|
extraPullNames: nixpkgs-ci
|
|
authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
|
|
pushFilter: '(-source|-single-chunk)$'
|
|
|
|
- name: Evaluate the ${{ matrix.system }} output paths at the merge commit
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }}
|
|
run: |
|
|
nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.singleSystem \
|
|
--argstr evalSystem "$MATRIX_SYSTEM" \
|
|
--arg chunkSize 8000 \
|
|
--argstr nixPath "$MATRIX_VERSION" \
|
|
--out-link merged
|
|
# If it uses too much memory, slightly decrease chunkSize.
|
|
# Note: Keep the same further down in sync!
|
|
|
|
# Running the attrpath generation step separately from the outpath step afterwards.
|
|
# The idea is that, *if* Eval on the target branch has not finished, yet, we will
|
|
# generate the attrpaths in the meantime - and the separate command command afterwards
|
|
# will check cachix again for whether Eval has finished. If no Eval result from the
|
|
# target branch can be found the second time, we proceed to run it in here. Attrpaths
|
|
# generation takes roughly 30 seconds, so for every normal use-case this should be more
|
|
# than enough of a head start for Eval on the target branch to finish.
|
|
# This edge-case, that Eval on the target branch is delayed is unlikely to happen anyway:
|
|
# For a commit to become the target commit of a PR, it must *already* be on the branch.
|
|
# Normally, CI should always start running on that push event *before* it starts running
|
|
# on the PR.
|
|
- name: Evaluate the ${{ matrix.system }} attribute paths at the target commit
|
|
if: inputs.targetSha
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
run: |
|
|
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.attrpathsSuperset \
|
|
--argstr evalSystem "$MATRIX_SYSTEM" \
|
|
--argstr nixPath "nixVersions.latest"
|
|
|
|
- name: Evaluate the ${{ matrix.system }} output paths at the target commit
|
|
if: inputs.targetSha
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
# This should be very quick, because it pulls the eval results from Cachix.
|
|
run: |
|
|
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.singleSystem \
|
|
--argstr evalSystem "$MATRIX_SYSTEM" \
|
|
--arg chunkSize 8000 \
|
|
--argstr nixPath "nixVersions.latest" \
|
|
--out-link target
|
|
|
|
- name: Compare outpaths against the target branch
|
|
if: inputs.targetSha
|
|
env:
|
|
MATRIX_SYSTEM: ${{ matrix.system }}
|
|
run: |
|
|
nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.diff \
|
|
--arg beforeDir ./target \
|
|
--arg afterDir ./merged \
|
|
--argstr evalSystem "$MATRIX_SYSTEM" \
|
|
--out-link diff
|
|
|
|
- name: Upload outpaths diff and stats
|
|
if: inputs.targetSha
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
with:
|
|
name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }}
|
|
path: diff/*
|
|
|
|
compare:
|
|
runs-on: ubuntu-24.04-arm
|
|
needs: [eval]
|
|
if: inputs.targetSha && !cancelled() && !failure()
|
|
permissions:
|
|
statuses: write
|
|
timeout-minutes: 5
|
|
steps:
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
sparse-checkout: .github/actions
|
|
- name: Check out the PR at the target commit
|
|
uses: ./.github/actions/checkout
|
|
with:
|
|
merged-as-untrusted-at: ${{ inputs.mergedSha }}
|
|
target-as-trusted-at: ${{ inputs.targetSha }}
|
|
|
|
- name: Download output paths and eval stats for all systems
|
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
|
|
with:
|
|
pattern: diff-*
|
|
path: diff
|
|
merge-multiple: true
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31
|
|
|
|
- name: Combine all output paths and eval stats
|
|
run: |
|
|
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.combine \
|
|
--arg diffDir ./diff \
|
|
--out-link combined
|
|
|
|
- name: Compare against the target branch
|
|
env:
|
|
AUTHOR_ID: ${{ github.event.pull_request.user.id }}
|
|
run: |
|
|
git -C nixpkgs/trusted diff --name-only ${{ inputs.mergedSha }} \
|
|
| jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json
|
|
|
|
# Use the target branch to get accurate maintainer info
|
|
nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.compare \
|
|
--arg combinedDir ./combined \
|
|
--arg touchedFilesJson ./touched-files.json \
|
|
--argstr githubAuthorId "$AUTHOR_ID" \
|
|
--out-link comparison
|
|
|
|
cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"
|
|
|
|
- name: Upload the comparison results
|
|
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
with:
|
|
name: comparison
|
|
path: comparison/*
|
|
|
|
- name: Add eval summary to commit statuses
|
|
if: ${{ github.event_name == 'pull_request_target' }}
|
|
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
|
|
with:
|
|
script: |
|
|
const { readFile } = require('node:fs/promises')
|
|
const changed = JSON.parse(await readFile('comparison/changed-paths.json', 'utf-8'))
|
|
const description =
|
|
'Package: ' + [
|
|
`added ${changed.attrdiff.added.length}`,
|
|
`removed ${changed.attrdiff.removed.length}`,
|
|
`changed ${changed.attrdiff.changed.length}`
|
|
].join(', ') +
|
|
' — Rebuild: ' + [
|
|
`linux ${changed.rebuildCountByKernel.linux}`,
|
|
`darwin ${changed.rebuildCountByKernel.darwin}`
|
|
].join(', ')
|
|
|
|
const { serverUrl, repo, runId, payload } = context
|
|
const target_url =
|
|
`${serverUrl}/${repo.owner}/${repo.repo}/actions/runs/${runId}?pr=${payload.pull_request.number}`
|
|
|
|
await github.rest.repos.createCommitStatus({
|
|
...repo,
|
|
sha: payload.pull_request.head.sha,
|
|
context: 'Eval Summary',
|
|
state: 'success',
|
|
description,
|
|
target_url
|
|
})
|
|
|
|
# Creates a matrix of Eval performance for various versions and systems.
|
|
report:
|
|
runs-on: ubuntu-24.04-arm
|
|
needs: [versions, eval]
|
|
steps:
|
|
- name: Download output paths and eval stats for all versions
|
|
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
|
|
with:
|
|
pattern: "*-diff-*"
|
|
path: versions
|
|
|
|
- name: Add version comparison table to job summary
|
|
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
|
|
env:
|
|
SYSTEMS: ${{ inputs.systems }}
|
|
VERSIONS: ${{ needs.versions.outputs.versions }}
|
|
with:
|
|
script: |
|
|
const { readFileSync } = require('node:fs')
|
|
const path = require('node:path')
|
|
|
|
const systems = JSON.parse(process.env.SYSTEMS)
|
|
const versions = JSON.parse(process.env.VERSIONS)
|
|
|
|
core.summary.addHeading('Lix/Nix version comparison')
|
|
core.summary.addTable(
|
|
[].concat(
|
|
[
|
|
[{ data: 'Version', header: true }].concat(
|
|
systems.map((system) => ({ data: system, header: true })),
|
|
),
|
|
],
|
|
versions.map((version) =>
|
|
[{ data: version }].concat(
|
|
systems.map((system) => {
|
|
try {
|
|
const artifact = path.join('versions', `${version}-diff-${system}`)
|
|
const time = Math.round(
|
|
parseFloat(
|
|
readFileSync(
|
|
path.join(artifact, 'after', system, 'total-time'),
|
|
'utf-8',
|
|
),
|
|
),
|
|
)
|
|
const diff = JSON.parse(
|
|
readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'),
|
|
)
|
|
const attrs = [].concat(
|
|
diff.added,
|
|
diff.removed,
|
|
diff.changed,
|
|
diff.rebuilds
|
|
).filter(attr =>
|
|
// Exceptions related to dev shells, which changed at some time between 2.18 and 2.24.
|
|
!attr.startsWith('tests.devShellTools.nixos.') &&
|
|
!attr.startsWith('tests.devShellTools.unstructuredDerivationInputEnv.')
|
|
)
|
|
if (attrs.length > 0) {
|
|
core.setFailed(
|
|
`${version} on ${system} has changed outpaths!\nNote: Please make sure to update ci/pinned.json separately from changes to other packages.`,
|
|
)
|
|
return { data: ':x:' }
|
|
}
|
|
return { data: time }
|
|
} catch {
|
|
core.warning(`${version} on ${system} did not produce artifact.`)
|
|
return { data: ':warning:' }
|
|
}
|
|
}),
|
|
),
|
|
),
|
|
),
|
|
)
|
|
core.summary.addRaw(
|
|
'\n*Evaluation time in seconds without downloading dependencies.*',
|
|
true,
|
|
)
|
|
core.summary.addRaw('\n*:warning: Job did not report a result.*', true)
|
|
core.summary.addRaw(
|
|
'\n*:x: Job produced different outpaths than the target branch.*',
|
|
true,
|
|
)
|
|
core.summary.write()
|
|
|
|
misc:
|
|
if: ${{ github.event_name != 'push' }}
|
|
runs-on: ubuntu-24.04-arm
|
|
timeout-minutes: 10
|
|
steps:
|
|
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
|
|
with:
|
|
sparse-checkout: .github/actions
|
|
- name: Checkout the merge commit
|
|
uses: ./.github/actions/checkout
|
|
with:
|
|
merged-as-untrusted-at: ${{ inputs.mergedSha }}
|
|
|
|
- name: Install Nix
|
|
uses: cachix/install-nix-action@7be5dee1421f63d07e71ce6e0a9f8a4b07c2a487 # v31
|
|
|
|
- name: Run misc eval tasks in parallel
|
|
run: |
|
|
# Ensure flake outputs on all systems still evaluate
|
|
nix flake check --all-systems --no-build './nixpkgs/untrusted?shallow=1' &
|
|
# Query nixpkgs with aliases enabled to check for basic syntax errors
|
|
nix-env -I ./nixpkgs/untrusted -f ./nixpkgs/untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null &
|
|
wait
|