From 6c105942cad57e0a4500b0195b478b3d9dfe8418 Mon Sep 17 00:00:00 2001
From: Wolfgang Walther <walther@technowledgy.de>
Date: Wed, 10 Sep 2025 13:46:25 +0200
Subject: [PATCH] ci/eval: fix local comparison with baseline

Due to how we pass in existing store paths via CLI arguments for the
diff and combine scripts, Nix didn't register a dependency on the store
paths properly. This meant that some of the derivations that were built,
didn't have the right store paths made available in the sandbox -
leading to all kinds of "not found" errors.

We worked around this in CI by resolving the symlinks to the nix store
beforehand. We tried to work around this locally by storing the nix
store path in BASELINE, but this didn't fully work. By explicitly
registering these store paths as dependencies, this should work across
the board - without any magic required by the caller.

(cherry picked from commit 45a765a28251a18f9e8bc695063c9ed6f28bd0b3)
---
 .github/workflows/eval.yml  |  6 +++---
 ci/eval/README.md           |  4 ++--
 ci/eval/compare/default.nix |  8 +++++---
 ci/eval/diff.nix            | 11 +++++++----
 4 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml
index 930426a933c9..eaf1ac35bb6e 100644
--- a/.github/workflows/eval.yml
+++ b/.github/workflows/eval.yml
@@ -153,8 +153,8 @@ jobs:
           MATRIX_SYSTEM: ${{ matrix.system }}
         run: |
           nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.diff \
-            --arg beforeDir "$(readlink ./target)" \
-            --arg afterDir "$(readlink ./merged)" \
+            --arg beforeDir ./target \
+            --arg afterDir ./merged \
             --argstr evalSystem "$MATRIX_SYSTEM" \
             --out-link diff
 
@@ -207,7 +207,7 @@ jobs:
 
           # Use the target branch to get accurate maintainer info
           nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.compare \
-            --arg combinedDir "$(realpath ./combined)" \
+            --arg combinedDir ./combined \
             --arg touchedFilesJson ./touched-files.json \
             --argstr githubAuthorId "$AUTHOR_ID" \
             --out-link comparison
diff --git a/ci/eval/README.md b/ci/eval/README.md
index d29ff94eb4a3..1cbb400552c7 100644
--- a/ci/eval/README.md
+++ b/ci/eval/README.md
@@ -33,13 +33,13 @@ Note that 16GB memory is the recommended minimum, while with less than 8GB memor
 To compare two commits locally, first run the following on the baseline commit:
 
 ```
-BASELINE=$(nix-build ci -A eval.baseline --no-out-link)
+nix-build ci -A eval.baseline --out-link baseline
 ```
 
 Then, on the commit with your changes:
 
 ```
-nix-build ci -A eval.full --arg baseline $BASELINE
+nix-build ci -A eval.full --arg baseline ./baseline
 ```
 
 Keep in mind to otherwise pass the same set of arguments for both commands (`evalSystems`, `quickTest`, `chunkSize`).
diff --git a/ci/eval/compare/default.nix b/ci/eval/compare/default.nix
index f5c1b1b1dab0..293877036e2f 100644
--- a/ci/eval/compare/default.nix
+++ b/ci/eval/compare/default.nix
@@ -13,6 +13,8 @@
   byName ? false,
 }:
 let
+  combined = builtins.storePath combinedDir;
+
   /*
     Derivation that computes which packages are affected (added, changed or removed) between two revisions of nixpkgs.
     Note: "platforms" are "x86_64-linux", "aarch64-darwin", ...
@@ -75,7 +77,7 @@ let
   # Attrs
   # - keys: "added", "changed", "removed" and "rebuilds"
   # - values: lists of `packagePlatformPath`s
-  diffAttrs = builtins.fromJSON (builtins.readFile "${combinedDir}/combined-diff.json");
+  diffAttrs = builtins.fromJSON (builtins.readFile "${combined}/combined-diff.json");
 
   changedPackagePlatformAttrs = convertToPackagePlatformAttrs diffAttrs.changed;
   rebuildsPackagePlatformAttrs = convertToPackagePlatformAttrs diffAttrs.rebuilds;
@@ -139,8 +141,8 @@ runCommand "compare"
     maintainers = builtins.toJSON maintainers;
     passAsFile = [ "maintainers" ];
     env = {
-      BEFORE_DIR = "${combinedDir}/before";
-      AFTER_DIR = "${combinedDir}/after";
+      BEFORE_DIR = "${combined}/before";
+      AFTER_DIR = "${combined}/after";
     };
   }
   ''
diff --git a/ci/eval/diff.nix b/ci/eval/diff.nix
index 785b0d784308..d22090601d30 100644
--- a/ci/eval/diff.nix
+++ b/ci/eval/diff.nix
@@ -11,6 +11,9 @@
 }:
 
 let
+  before = builtins.storePath beforeDir;
+  after = builtins.storePath afterDir;
+
   /*
     Computes the key difference between two attrs
 
@@ -64,15 +67,15 @@ let
     in
     builtins.fromJSON data;
 
-  beforeAttrs = getAttrs beforeDir;
-  afterAttrs = getAttrs afterDir;
+  beforeAttrs = getAttrs before;
+  afterAttrs = getAttrs after;
   diffAttrs = diff beforeAttrs afterAttrs;
   diffJson = writeText "diff.json" (builtins.toJSON diffAttrs);
 in
 runCommand "diff" { } ''
   mkdir -p $out/${evalSystem}
 
-  cp -r ${beforeDir} $out/before
-  cp -r ${afterDir} $out/after
+  cp -r ${before} $out/before
+  cp -r ${after} $out/after
   cp ${diffJson} $out/${evalSystem}/diff.json
 ''