name: Eval

on:
  workflow_call:
    inputs:
      mergedSha:
        required: true
        type: string
      targetSha:
        type: string
      systems:
        required: true
        type: string
      testVersions:
        required: false
        default: false
        type: boolean
    secrets:
      OWNER_APP_PRIVATE_KEY:
        required: false

permissions: {}

defaults:
  run:
    shell: bash

jobs:
  versions:
    if: inputs.testVersions
    runs-on: ubuntu-24.04-arm
    outputs:
      versions: ${{ steps.versions.outputs.versions }}
    steps:
      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          path: trusted
          sparse-checkout: |
            ci/supportedVersions.nix

      - name: Check out the PR at the test merge commit
        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          ref: ${{ inputs.mergedSha }}
          path: untrusted
          sparse-checkout: |
            ci/pinned.json

      - name: Install Nix
        uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31

      - name: Load supported versions
        id: versions
        run: |
          echo "versions=$(trusted/ci/supportedVersions.nix --arg pinnedJson untrusted/ci/pinned.json)" >> "$GITHUB_OUTPUT"

  eval:
    runs-on: ubuntu-24.04-arm
    needs: versions
    if: ${{ !cancelled() }}
    strategy:
      fail-fast: false
      matrix:
        system: ${{ fromJSON(inputs.systems) }}
        version:
          - "" # Default Eval triggering rebuild labels and such.
          - ${{ fromJSON(needs.versions.outputs.versions || '[]') }} # Only for ci/pinned.json updates.
    # Failures for versioned Evals will be collected in a separate job below
    # to not interrupt main Eval's compare step.
    continue-on-error: ${{ matrix.version != '' }}
    name: ${{ matrix.system }}${{ matrix.version && format(' @ {0}', matrix.version) || '' }}
    outputs:
      targetRunId: ${{ steps.targetRunId.outputs.targetRunId }}
    timeout-minutes: 15
    steps:
      # This is not supposed to be used and just acts as a fallback.
      # Without swap, when Eval runs OOM, it will fail badly with a
      # job that is sometimes not interruptible anymore.
      # If Eval starts swapping, decrease chunkSize to keep it fast.
      - name: Enable swap
        run: |
          sudo fallocate -l 10G /swap
          sudo chmod 600 /swap
          sudo mkswap /swap
          sudo swapon /swap

      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          sparse-checkout: .github/actions
      - name: Check out the PR at the test merge commit
        uses: ./.github/actions/get-merge-commit
        with:
          mergedSha: ${{ inputs.mergedSha }}
          merged-as-untrusted: true
          pinnedFrom: untrusted

      - name: Install Nix
        uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31

      - name: Evaluate the ${{ matrix.system }} output paths for all derivation attributes
        env:
          MATRIX_SYSTEM: ${{ matrix.system }}
          MATRIX_VERSION: ${{ matrix.version || 'nixVersions.latest' }}
        run: |
          nix-build untrusted/ci --arg nixpkgs ./pinned  -A eval.singleSystem \
            --argstr evalSystem "$MATRIX_SYSTEM" \
            --arg chunkSize 8000 \
            --argstr nixPath "$MATRIX_VERSION" \
            --out-link merged
          # If it uses too much memory, slightly decrease chunkSize

      - name: Upload the output paths and eval stats
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}merged-${{ matrix.system }}
          path: merged/*

      - name: Log current API rate limits
        env:
          GH_TOKEN: ${{ github.token }}
        run: gh api /rate_limit | jq

      - name: Get target run id
        if: inputs.targetSha
        id: targetRunId
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
        env:
          MATRIX_SYSTEM: ${{ matrix.system }}
          TARGET_SHA: ${{ inputs.targetSha }}
        with:
          script: |
            const system = process.env.MATRIX_SYSTEM
            const targetSha = process.env.TARGET_SHA

            let run_id
            try {
              run_id = (await github.rest.actions.listWorkflowRuns({
                ...context.repo,
                workflow_id: 'push.yml',
                event: 'push',
                head_sha: targetSha
              })).data.workflow_runs[0].id
            } catch {
              throw new Error(`Could not find a push.yml workflow run for ${targetSha}.`)
            }

            // Waiting 120 * 5 sec = 10 min. max.
            // Eval takes max 5-6 minutes, normally.
            for (let i = 0; i < 120; i++) {
              const result = await github.rest.actions.listWorkflowRunArtifacts({
                ...context.repo,
                run_id,
                name: `merged-${system}`
              })
              if (result.data.total_count > 0) {
                core.setOutput('targetRunId', run_id)
                return
              }
              await new Promise(resolve => setTimeout(resolve, 5000))
            }
            // No artifact found at this stage. This usually means that Eval failed on the target branch.
            // This should only happen when Eval is broken on the target branch and this PR fixes it.
            // Continue without targetRunId to skip the remaining steps, but pass the job.

      - name: Log current API rate limits
        env:
          GH_TOKEN: ${{ github.token }}
        run: gh api /rate_limit | jq

      - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
        if: steps.targetRunId.outputs.targetRunId
        with:
          run-id: ${{ steps.targetRunId.outputs.targetRunId }}
          name: merged-${{ matrix.system }}
          path: target
          github-token: ${{ github.token }}
          merge-multiple: true

      - name: Compare outpaths against the target branch
        if: steps.targetRunId.outputs.targetRunId
        env:
          MATRIX_SYSTEM: ${{ matrix.system }}
        run: |
          nix-build untrusted/ci --arg nixpkgs ./pinned -A eval.diff \
            --arg beforeDir ./target \
            --arg afterDir "$(readlink ./merged)" \
            --argstr evalSystem "$MATRIX_SYSTEM" \
            --out-link diff

      - name: Upload outpaths diff and stats
        if: steps.targetRunId.outputs.targetRunId
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: ${{ matrix.version && format('{0}-', matrix.version) || '' }}diff-${{ matrix.system }}
          path: diff/*

  compare:
    runs-on: ubuntu-24.04-arm
    needs: [eval]
    if: needs.eval.outputs.targetRunId
    permissions:
      statuses: write
    timeout-minutes: 5
    steps:
      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          sparse-checkout: .github/actions
      - name: Check out the PR at the target commit
        uses: ./.github/actions/get-merge-commit
        with:
          targetSha: ${{ inputs.targetSha }}
          target-as-trusted: true
          pinnedFrom: trusted

      - name: Download output paths and eval stats for all systems
        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
        with:
          pattern: diff-*
          path: diff
          merge-multiple: true

      - name: Install Nix
        uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31

      - name: Combine all output paths and eval stats
        run: |
          nix-build trusted/ci --arg nixpkgs ./pinned -A eval.combine \
            --arg diffDir ./diff \
            --out-link combined

      - name: Compare against the target branch
        env:
          AUTHOR_ID: ${{ github.event.pull_request.user.id }}
        run: |
          git -C trusted fetch --depth 1 origin ${{ inputs.mergedSha }}
          git -C trusted diff --name-only ${{ inputs.mergedSha }} \
            | jq --raw-input --slurp 'split("\n")[:-1]' > touched-files.json

          # Use the target branch to get accurate maintainer info
          nix-build trusted/ci --arg nixpkgs ./pinned -A eval.compare \
            --arg combinedDir "$(realpath ./combined)" \
            --arg touchedFilesJson ./touched-files.json \
            --argstr githubAuthorId "$AUTHOR_ID" \
            --out-link comparison

          cat comparison/step-summary.md >> "$GITHUB_STEP_SUMMARY"

      - name: Upload the comparison results
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: comparison
          path: comparison/*

      - name: Add eval summary to commit statuses
        if: ${{ github.event_name == 'pull_request_target' }}
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
        with:
          script: |
            const { readFile } = require('node:fs/promises')
            const changed = JSON.parse(await readFile('comparison/changed-paths.json', 'utf-8'))
            const description =
              'Package: ' + [
                `added ${changed.attrdiff.added.length}`,
                `removed ${changed.attrdiff.removed.length}`,
                `changed ${changed.attrdiff.changed.length}`
              ].join(', ') +
              ' — Rebuild: ' + [
                `linux ${changed.rebuildCountByKernel.linux}`,
                `darwin ${changed.rebuildCountByKernel.darwin}`
              ].join(', ')

            const { serverUrl, repo, runId, payload } = context
            const target_url =
              `${serverUrl}/${repo.owner}/${repo.repo}/actions/runs/${runId}?pr=${payload.pull_request.number}`

            await github.rest.repos.createCommitStatus({
              ...repo,
              sha: payload.pull_request.head.sha,
              context: 'Eval Summary',
              state: 'success',
              description,
              target_url
            })

  # Creates a matrix of Eval performance for various versions and systems.
  report:
    runs-on: ubuntu-24.04-arm
    needs: [versions, eval]
    steps:
      - name: Download output paths and eval stats for all versions
        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
        with:
          pattern: "*-diff-*"
          path: versions

      - name: Add version comparison table to job summary
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
        env:
          SYSTEMS: ${{ inputs.systems }}
          VERSIONS: ${{ needs.versions.outputs.versions }}
        with:
          script: |
            const { readFileSync } = require('node:fs')
            const path = require('node:path')

            const systems = JSON.parse(process.env.SYSTEMS)
            const versions = JSON.parse(process.env.VERSIONS)

            core.summary.addHeading('Lix/Nix version comparison')
            core.summary.addTable(
              [].concat(
                [
                  [{ data: 'Version', header: true }].concat(
                    systems.map((system) => ({ data: system, header: true })),
                  ),
                ],
                versions.map((version) =>
                  [{ data: version }].concat(
                    systems.map((system) => {
                      try {
                        const artifact = path.join('versions', `${version}-diff-${system}`)
                        const time = Math.round(
                          parseFloat(
                            readFileSync(
                              path.join(artifact, 'after', system, 'total-time'),
                              'utf-8',
                            ),
                          ),
                        )
                        const diff = JSON.parse(
                          readFileSync(path.join(artifact, system, 'diff.json'), 'utf-8'),
                        )
                        const attrs = [].concat(
                          diff.added,
                          diff.removed,
                          diff.changed,
                          diff.rebuilds
                        ).filter(attr =>
                          // Exceptions related to dev shells, which changed at some time between 2.18 and 2.24.
                          !attr.startsWith('tests.devShellTools.nixos.') &&
                          !attr.startsWith('tests.devShellTools.unstructuredDerivationInputEnv.')
                        )
                        if (attrs.length > 0) {
                          core.setFailed(
                            `${version} on ${system} has changed outpaths!\nNote: Please make sure to update ci/pinned.json separately from changes to other packages.`,
                          )
                          return { data: ':x:' }
                        }
                        return { data: time }
                      } catch {
                        core.warning(`${version} on ${system} did not produce artifact.`)
                        return { data: ':warning:' }
                      }
                    }),
                  ),
                ),
              ),
            )
            core.summary.addRaw(
              '\n*Evaluation time in seconds without downloading dependencies.*',
              true,
            )
            core.summary.addRaw('\n*:warning: Job did not report a result.*', true)
            core.summary.addRaw(
              '\n*:x: Job produced different outpaths than the target branch.*',
              true,
            )
            core.summary.write()

  misc:
    if: ${{ github.event_name != 'push' }}
    runs-on: ubuntu-24.04-arm
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          sparse-checkout: .github/actions
      - name: Check if the PR can be merged and checkout the merge commit
        uses: ./.github/actions/get-merge-commit
        with:
          merged-as-untrusted: true

      - name: Install Nix
        uses: cachix/install-nix-action@fc6e360bedc9ee72d75e701397f0bb30dce77568 # v31

      - name: Ensure flake outputs on all systems still evaluate
        run: nix flake check --all-systems --no-build ./untrusted

      - name: Query nixpkgs with aliases enabled to check for basic syntax errors
        run: |
          time nix-env -I ./untrusted -f ./untrusted -qa '*' --option restrict-eval true --option allow-import-from-derivation false >/dev/null