ci/eval.full: allow local comparison with rebuilds

This allows running a full comparison between two commits locally.

What was previously `eval.full` is now called `eval.all`. The new
`eval.full` takes a `baseline` argument for the comparison.

(cherry picked from commit ccc12c839b)

ci/eval/README.md

@@ -5,7 +5,7 @@ The code in this directory is used by the [eval.yml](../../.github/workflows/eva
 Furthermore it also allows local evaluation using:
 
 ```
-nix-build ci -A eval.full
+nix-build ci -A eval.baseline
 ```
 
 The most important two arguments are:
@@ -27,3 +27,19 @@ The following arguments can be used to fine-tune performance:
   Example: `--arg chunkSize 10000`
 
 Note that 16GB memory is the recommended minimum, while with less than 8GB memory evaluation time suffers greatly.
+
+## Local eval with rebuilds / comparison
+
+To compare two commits locally, first run the following on the baseline commit:
+
+```
+BASELINE=$(nix-build ci -A eval.baseline --no-out-link)
+```
+
+Then, on the commit with your changes:
+
+```
+nix-build ci -A eval.full --arg baseline $BASELINE
+```
+
+Keep in mind to otherwise pass the same set of arguments for both commands (`evalSystems`, `quickTest`, `chunkSize`).

ci/eval/default.nix

@@ -240,7 +240,7 @@ let
 
   compare = callPackage ./compare { };
 
-  full =
+  baseline =
     {
       # Whether to evaluate on a specific set of systems, by default all are evaluated
       evalSystems ? if quickTest then [ "x86_64-linux" ] else supportedSystems,
@@ -248,21 +248,36 @@ let
       chunkSize ? 5000,
       quickTest ? false,
     }:
+    symlinkJoin {
+      name = "nixpkgs-eval-baseline";
+      paths = map (
+        evalSystem:
+        singleSystem {
+          inherit quickTest evalSystem chunkSize;
+        }
+      ) evalSystems;
+    };
+
+  full =
+    {
+      # Whether to evaluate on a specific set of systems, by default all are evaluated
+      evalSystems ? if quickTest then [ "x86_64-linux" ] else supportedSystems,
+      # The number of attributes per chunk, see ./README.md for more info.
+      chunkSize ? 5000,
+      quickTest ? false,
+      baseline,
+    }:
     let
       diffs = symlinkJoin {
-        name = "diffs";
+        name = "nixpkgs-eval-diffs";
         paths = map (
           evalSystem:
-          let
-            eval = singleSystem {
-              inherit quickTest evalSystem chunkSize;
-            };
-          in
           diff {
             inherit evalSystem;
-            # Local "full" evaluation doesn't do a real diff.
-            beforeDir = eval;
-            afterDir = eval;
+            beforeDir = baseline;
+            afterDir = singleSystem {
+              inherit quickTest evalSystem chunkSize;
+            };
           }
         ) evalSystems;
       };
@@ -280,7 +295,8 @@ in
     combine
     compare
     # The above three are used by separate VMs in a GitHub workflow,
-    # while the below is intended for testing on a single local machine
+    # while the below are intended for testing on a single local machine
+    baseline
     full
     ;
 }