From c40c306541b3cd9690513089f049927be9844e0e Mon Sep 17 00:00:00 2001
From: Aliaksandr <grubian2@gmail.com>
Date: Mon, 25 May 2026 11:04:25 +0300
Subject: [PATCH] ci/eval/compare: show performance comparison even when
 package sets differ

Previously the eval comparison was skipped entirely when packages were
added or removed between revisions, leaving only a "stats were skipped"
message. cmp-stats already joins on the stats chunks present in both
revisions, so a comparison can still be produced. Always run it and add
a GitHub note caveating that the figures are approximate when the
package sets differ.

Assisted-by: claude-code with claude-opus-4-7[1m]-xhigh
(cherry picked from commit 25f9394b8a6cb5273d8650bfea949e8f690dc12d)
---
 ci/eval/compare/default.nix | 43 ++++++++++++++++++-------------------
 1 file changed, 21 insertions(+), 22 deletions(-)

diff --git a/ci/eval/compare/default.nix b/ci/eval/compare/default.nix
index e596b6d1994f..856969ee8695 100644
--- a/ci/eval/compare/default.nix
+++ b/ci/eval/compare/default.nix
@@ -238,34 +238,33 @@ runCommand "compare"
       jq -r -f ${./generate-step-summary.jq} < ${changed-paths}
     } >> $out/step-summary.md
 
-    if jq -e '(.attrdiff.added | length == 0) and (.attrdiff.removed | length == 0)' "${changed-paths}" > /dev/null; then
-      # Chunks have changed between revisions
-      # We cannot generate a performance comparison
-      {
-        echo
-        echo "# Performance comparison"
-        echo
-        echo "This compares the performance of this branch against its pull request base branch (e.g., 'master')"
-        echo
-        echo "For further help please refer to: [ci/README.md](https://github.com/NixOS/nixpkgs/blob/master/ci/README.md)"
-        echo
-      } >> $out/step-summary.md
+    {
+      echo
+      echo "# Performance comparison"
+      echo
+      echo "This compares the performance of this branch against its pull request base branch (e.g., 'master')"
+      echo
+    } >> $out/step-summary.md
 
-      cmp-stats --explain ${combined}/before/stats ${combined}/after/stats >> $out/step-summary.md
-
-    else
-      # Package chunks are the same in both revisions
-      # We can use the to generate a performance comparison
+    # cmp-stats only compares the stats chunks present in both revisions, so the
+    # comparison is still produced when packages were added/removed. The paired
+    # chunks may cover different attrs in that case, so caveat the figures.
+    if ! jq -e '(.attrdiff.added | length == 0) and (.attrdiff.removed | length == 0)' "${changed-paths}" > /dev/null; then
       {
+        echo "> [!NOTE]"
+        echo "> The package sets differ between the two revisions. This comparison only"
+        echo "> covers packages evaluated in both, so treat the figures as approximate."
         echo
-        echo "# Performance Comparison"
-        echo
-        echo "Performance stats were skipped because the package sets differ between the two revisions."
-        echo
-        echo "For further help please refer to: [ci/README.md](https://github.com/NixOS/nixpkgs/blob/master/ci/README.md)"
       } >> $out/step-summary.md
     fi
 
+    {
+      echo "For further help please refer to: [ci/README.md](https://github.com/NixOS/nixpkgs/blob/master/ci/README.md)"
+      echo
+    } >> $out/step-summary.md
+
+    cmp-stats --explain ${combined}/before/stats ${combined}/after/stats >> $out/step-summary.md
+
     jq -r '.[]' "${touchedFilesJson}" > ./touched-files
     readarray -t touchedFiles < ./touched-files
     echo "This PR touches ''${#touchedFiles[@]} files"