nixpkgs/.github/workflows/eval.yml

name: Eval

on:
  pull_request:
    paths:
      - .github/workflows/eval.yml
  pull_request_target:
    types: [opened, ready_for_review, synchronize, reopened]
  push:
    # Keep this synced with ci/request-reviews/dev-branches.txt
    branches:
      - master
      - staging
      - release-*
      - staging-*
      - haskell-updates
      - python-updates

permissions: {}

jobs:
  prepare:
    name: Prepare
    runs-on: ubuntu-24.04-arm
    outputs:
      mergedSha: ${{ steps.get-merge-commit.outputs.mergedSha }}
      targetSha: ${{ steps.get-merge-commit.outputs.targetSha }}
      systems: ${{ steps.systems.outputs.systems }}
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          sparse-checkout: |
            .github/actions
            ci/supportedSystems.json
      - name: Check if the PR can be merged and get the test merge commit
        uses: ./.github/actions/get-merge-commit
        id: get-merge-commit

      - name: Load supported systems
        id: systems
        run: |
          echo "systems=$(jq -c <ci/supportedSystems.json)" >> "$GITHUB_OUTPUT"

  outpaths:
    name: Outpaths
    runs-on: ubuntu-24.04-arm
    needs: [ prepare ]
    strategy:
      fail-fast: false
      matrix:
        system: ${{ fromJSON(needs.prepare.outputs.systems) }}
    steps:
      - name: Enable swap
        run: |
          sudo fallocate -l 10G /swap
          sudo chmod 600 /swap
          sudo mkswap /swap
          sudo swapon /swap

      - name: Check out the PR at the test merge commit
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: ${{ needs.prepare.outputs.mergedSha }}
          path: untrusted

      - name: Install Nix
        uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
        with:
          extra_nix_config: sandbox = true

      - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes
        env:
          MATRIX_SYSTEM: ${{ matrix.system }}
        run: |
          nix-build untrusted/ci -A eval.singleSystem \
            --argstr evalSystem "$MATRIX_SYSTEM" \
            --arg chunkSize 10000 \
            --out-link merged
          # If it uses too much memory, slightly decrease chunkSize

      - name: Upload the output paths and eval stats
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: merged-${{ matrix.system }}
          path: merged/*

      - name: Get target run id
        if: needs.prepare.outputs.targetSha
        id: targetRunId
        env:
          GH_TOKEN: ${{ github.token }}
          MATRIX_SYSTEM: ${{ matrix.system }}
          REPOSITORY: ${{ github.repository }}
          TARGET_SHA: ${{ needs.prepare.outputs.targetSha }}
        run: |
          # Get the latest eval.yml workflow run for the PR's target commit
          if ! run=$(gh api --method GET /repos/"$REPOSITORY"/actions/workflows/eval.yml/runs \
            -f head_sha="$TARGET_SHA" -f event=push \
            --jq '.workflow_runs | sort_by(.run_started_at) | .[-1]') \
            || [[ -z "$run" ]]; then
            echo "Could not find an eval.yml workflow run for $TARGET_SHA, cannot make comparison"
            exit 1
          fi
          echo "Comparing against $(jq .html_url <<< "$run")"
          runId=$(jq .id <<< "$run")

          if ! job=$(gh api --method GET /repos/"$REPOSITORY"/actions/runs/"$runId"/jobs \
            --jq ".jobs[] | select (.name == \"Outpaths ($MATRIX_SYSTEM)\")") \
            || [[ -z "$job" ]]; then
            echo "Could not find the Outpaths ($MATRIX_SYSTEM) job for workflow run $runId, cannot make comparison"
            exit 1
          fi
          jobId=$(jq .id <<< "$job")
          conclusion=$(jq -r .conclusion <<< "$job")

          while [[ "$conclusion" == null || "$conclusion" == "" ]]; do
            echo "Job not done, waiting 10 seconds before checking again"
            sleep 10
            conclusion=$(gh api /repos/"$REPOSITORY"/actions/jobs/"$jobId" --jq '.conclusion')
          done

          if [[ "$conclusion" != "success" ]]; then
            echo "Job was not successful (conclusion: $conclusion), cannot make comparison"
            exit 1
          fi

          echo "targetRunId=$runId" >> "$GITHUB_OUTPUT"

      - uses: actions/download-artifact@v4
        if: steps.targetRunId.outputs.targetRunId
        with:
          run-id: ${{ steps.targetRunId.outputs.targetRunId }}
          name: merged-${{ matrix.system }}
          path: target
          github-token: ${{ github.token }}
          merge-multiple: true

      - name: Compare outpaths against the target branch
        if: steps.targetRunId.outputs.targetRunId
        env:
          MATRIX_SYSTEM: ${{ matrix.system }}
        run: |
          nix-build untrusted/ci -A eval.diff \
            --arg beforeDir ./target \
            --arg afterDir "$(readlink ./merged)" \
            --argstr evalSystem "$MATRIX_SYSTEM" \
            --out-link diff

      - name: Upload outpaths diff and stats
        if: steps.targetRunId.outputs.targetRunId
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: diff-${{ matrix.system }}
          path: diff/*

  tag:
    name: Tag
    runs-on: ubuntu-24.04-arm
    needs: [ prepare, outpaths ]
    if: needs.prepare.outputs.targetSha
    permissions:
      pull-requests: write
      statuses: write
    steps:
      - name: Download output paths and eval stats for all systems
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          pattern: diff-*
          path: diff
          merge-multiple: true

      - name: Check out the PR at the target commit
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          ref: ${{ needs.prepare.outputs.targetSha }}
          path: trusted

      - name: Install Nix
        uses: cachix/install-nix-action@526118121621777ccd86f79b04685a9319637641 # v31
        with:
          extra_nix_config: sandbox = true

      - name: Combine all output paths and eval stats
        run: |
          nix-build trusted/ci -A eval.combine \
            --arg diffDir ./diff \
            --out-link combined

      - name: Compare against the target branch
        env:
          AUTHOR_ID: ${{ github.event.pull_request.user.id }}
        run: |
          git -C trusted fetch --depth 1 origin ${{ needs.prepare.outputs.mergedSha }}
          git -C trusted diff --name-only ${{ needs.prepare.outputs.mergedSha }} \
            | jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json

          # Use the target branch to get accurate maintainer info
          nix-build trusted/ci -A eval.compare \
            --arg combinedDir "$(realpath ./combined)" \
            --arg touchedFilesJson ./touched-files.json \
            --argstr githubAuthorId "$AUTHOR_ID" \
            --out-link comparison

          cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"

      - name: Upload the comparison results
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: comparison
          path: comparison/*

      - name: Build the requestReviews derivation
        run: nix-build trusted/ci -A requestReviews

      - name: Labelling pull request
        if: ${{ github.event_name == 'pull_request_target' && github.repository_owner == 'NixOS' }}
        env:
          GH_TOKEN: ${{ github.token }}
          REPOSITORY: ${{ github.repository }}
          NUMBER: ${{ github.event.number }}
        run: |
          # Get all currently set labels that we manage
          gh api \
            /repos/"$REPOSITORY"/issues/"$NUMBER"/labels \
            --jq '.[].name | select(startswith("10.rebuild") or . == "11.by: package-maintainer")' \
            | sort > before

          # And the labels that should be there
          jq -r '.labels[]' comparison/changed-paths.json \
            | sort > after

          # Remove the ones not needed anymore
          while read -r toRemove; do
            echo "Removing label $toRemove"
            gh api \
              --method DELETE \
              /repos/"$REPOSITORY"/issues/"$NUMBER"/labels/"$toRemove"
          done < <(comm -23 before after)

          # And add the ones that aren't set already
          while read -r toAdd; do
            echo "Adding label $toAdd"
            gh api \
              --method POST \
              /repos/"$REPOSITORY"/issues/"$NUMBER"/labels \
              -f "labels[]=$toAdd"
          done < <(comm -13 before after)

      - name: Add eval summary to commit statuses
        if: ${{ github.event_name == 'pull_request_target' && github.repository_owner == 'NixOS' }}
        env:
          GH_TOKEN: ${{ github.token }}
          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
          NUMBER: ${{ github.event.number }}
        run: |
          description=$(jq -r '
          "Package: added " + (.attrdiff.added | length | tostring) +
          ", removed " + (.attrdiff.removed | length | tostring) +
          ", changed " + (.attrdiff.changed | length | tostring) +
          ", Rebuild: linux " + (.rebuildCountByKernel.linux | tostring) +
          ", darwin " + (.rebuildCountByKernel.darwin | tostring)
          ' <comparison/changed-paths.json)
          target_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID?pr=$NUMBER"
          gh api --method POST \
            -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" \
            "/repos/$GITHUB_REPOSITORY/statuses/$PR_HEAD_SHA" \
            -f "context=Eval / Summary" -f "state=success" -f "description=$description" -f "target_url=$target_url"

      # See ./codeowners-v2.yml, reuse the same App because we need the same permissions
      # Can't use the token received from permissions above, because it can't get enough permissions
      - uses: actions/create-github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e # v2.0.6
        if: vars.OWNER_APP_ID
        id: app-token
        with:
          app-id: ${{ vars.OWNER_APP_ID }}
          private-key: ${{ secrets.OWNER_APP_PRIVATE_KEY }}
          permission-administration: read
          permission-members: read
          permission-pull-requests: write

      - name: Requesting maintainer reviews
        if: ${{ steps.app-token.outputs.token && github.repository_owner == 'NixOS' }}
        env:
          GH_TOKEN: ${{ github.token }}
          REPOSITORY: ${{ github.repository }}
          NUMBER: ${{ github.event.number }}
          AUTHOR: ${{ github.event.pull_request.user.login }}
          # Don't request reviewers on draft PRs
          DRY_MODE: ${{ github.event.pull_request.draft && '1' || '' }}
        run: |
          # maintainers.json contains GitHub IDs. Look up handles to request reviews from.
          # There appears to be no API to request reviews based on GitHub IDs
          jq -r 'keys[]' comparison/maintainers.json \
            | while read -r id; do gh api /user/"$id" --jq .login; done \
            | GH_TOKEN=${{ steps.app-token.outputs.token }} result/bin/request-reviewers.sh "$REPOSITORY" "$NUMBER" "$AUTHOR"