ci/eval: fix local comparison with baseline

Due to how we pass in existing store paths via CLI arguments for the
diff and combine scripts, Nix didn't register a dependency on the store
paths properly. This meant that some of the derivations that were built,
didn't have the right store paths made available in the sandbox -
leading to all kinds of "not found" errors.

We worked around this in CI by resolving the symlinks to the nix store
beforehand. We tried to work around this locally by storing the nix
store path in BASELINE, but this didn't fully work. By explicitly
registering these store paths as dependencies, this should work across
the board - without any magic required by the caller.

.github/workflows/eval.yml

@@ -153,8 +153,8 @@ jobs:
           MATRIX_SYSTEM: ${{ matrix.system }}
         run: |
           nix-build nixpkgs/untrusted/ci --arg nixpkgs ./nixpkgs/untrusted-pinned -A eval.diff \
-            --arg beforeDir "$(readlink ./target)" \
-            --arg afterDir "$(readlink ./merged)" \
+            --arg beforeDir ./target \
+            --arg afterDir ./merged \
             --argstr evalSystem "$MATRIX_SYSTEM" \
             --out-link diff
 
@@ -207,7 +207,7 @@ jobs:
 
           # Use the target branch to get accurate maintainer info
           nix-build nixpkgs/trusted/ci --arg nixpkgs ./nixpkgs/trusted-pinned -A eval.compare \
-            --arg combinedDir "$(realpath ./combined)" \
+            --arg combinedDir ./combined \
             --arg touchedFilesJson ./touched-files.json \
             --argstr githubAuthorId "$AUTHOR_ID" \
             --out-link comparison

ci/eval/README.md

@@ -33,13 +33,13 @@ Note that 16GB memory is the recommended minimum, while with less than 8GB memor
 To compare two commits locally, first run the following on the baseline commit:
 
 ```
-BASELINE=$(nix-build ci -A eval.baseline --no-out-link)
+nix-build ci -A eval.baseline --out-link baseline
 ```
 
 Then, on the commit with your changes:
 
 ```
-nix-build ci -A eval.full --arg baseline $BASELINE
+nix-build ci -A eval.full --arg baseline ./baseline
 ```
 
 Keep in mind to otherwise pass the same set of arguments for both commands (`evalSystems`, `quickTest`, `chunkSize`).

ci/eval/compare/default.nix

@@ -13,6 +13,8 @@
   byName ? false,
 }:
 let
+  combined = builtins.storePath combinedDir;
+
   /*
     Derivation that computes which packages are affected (added, changed or removed) between two revisions of nixpkgs.
     Note: "platforms" are "x86_64-linux", "aarch64-darwin", ...
@@ -75,7 +77,7 @@ let
   # Attrs
   # - keys: "added", "changed", "removed" and "rebuilds"
   # - values: lists of `packagePlatformPath`s
-  diffAttrs = builtins.fromJSON (builtins.readFile "${combinedDir}/combined-diff.json");
+  diffAttrs = builtins.fromJSON (builtins.readFile "${combined}/combined-diff.json");
 
   changedPackagePlatformAttrs = convertToPackagePlatformAttrs diffAttrs.changed;
   rebuildsPackagePlatformAttrs = convertToPackagePlatformAttrs diffAttrs.rebuilds;
@@ -139,8 +141,8 @@ runCommand "compare"
     maintainers = builtins.toJSON maintainers;
     passAsFile = [ "maintainers" ];
     env = {
-      BEFORE_DIR = "${combinedDir}/before";
-      AFTER_DIR = "${combinedDir}/after";
+      BEFORE_DIR = "${combined}/before";
+      AFTER_DIR = "${combined}/after";
     };
   }
   ''

ci/eval/diff.nix

@@ -11,6 +11,9 @@
 }:
 
 let
+  before = builtins.storePath beforeDir;
+  after = builtins.storePath afterDir;
+
   /*
     Computes the key difference between two attrs
 
@@ -64,15 +67,15 @@ let
     in
     builtins.fromJSON data;
 
-  beforeAttrs = getAttrs beforeDir;
-  afterAttrs = getAttrs afterDir;
+  beforeAttrs = getAttrs before;
+  afterAttrs = getAttrs after;
   diffAttrs = diff beforeAttrs afterAttrs;
   diffJson = writeText "diff.json" (builtins.toJSON diffAttrs);
 in
 runCommand "diff" { } ''
   mkdir -p $out/${evalSystem}
 
-  cp -r ${beforeDir} $out/before
-  cp -r ${afterDir} $out/after
+  cp -r ${before} $out/before
+  cp -r ${after} $out/after
   cp ${diffJson} $out/${evalSystem}/diff.json
 ''