top-level/release-outpaths: move to ci/eval & refactor (#439221)

ci/eval/attrpaths.nix

@@ -0,0 +1,83 @@
+# This expression will, as efficiently as possible, dump a
+# *superset* of all attrpaths of derivations which might be
+# part of a release on *any* platform.
+#
+# This expression runs single-threaded under all current Nix
+# implementations, but much faster and with much less memory
+# used than ./outpaths.nix itself.
+#
+# Once you have the list of attrnames you can split it up into
+# $NUM_CORES batches and evaluate the outpaths separately for each
+# batch, in parallel.
+#
+# To dump the attrnames:
+#
+#   nix-instantiate --eval --strict --json ci/eval/attrpaths.nix -A names
+#
+{
+  lib ? import (path + "/lib"),
+  trace ? false,
+  path ? ./../..,
+}:
+let
+
+  # TODO: Use mapAttrsToListRecursiveCond when this PR lands:
+  # https://github.com/NixOS/nixpkgs/pull/395160
+  justAttrNames =
+    path: value:
+    let
+      result =
+        if path == [ "AAAAAASomeThingsFailToEvaluate" ] || !(lib.isAttrs value) then
+          [ ]
+        else if lib.isDerivation value then
+          [ path ]
+        else
+          lib.pipe value [
+            (lib.mapAttrsToList (
+              name: value:
+              lib.addErrorContext "while evaluating package set attribute path '${
+                lib.showAttrPath (path ++ [ name ])
+              }'" (justAttrNames (path ++ [ name ]) value)
+            ))
+            lib.concatLists
+          ];
+    in
+    lib.traceIf trace "** ${lib.showAttrPath path}" result;
+
+  outpaths = import ./outpaths.nix {
+    inherit path;
+    attrNamesOnly = true;
+  };
+
+  paths = [
+    # Some of the following are based on variants, which are disabled with `attrNamesOnly = true`.
+    # Until these have been removed from release.nix / hydra, we manually add them to the list.
+    [
+      "pkgsLLVM"
+      "stdenv"
+    ]
+    [
+      "pkgsArocc"
+      "stdenv"
+    ]
+    [
+      "pkgsZig"
+      "stdenv"
+    ]
+    [
+      "pkgsStatic"
+      "stdenv"
+    ]
+    [
+      "pkgsMusl"
+      "stdenv"
+    ]
+  ]
+  ++ justAttrNames [ ] outpaths;
+
+  names = map lib.showAttrPath paths;
+
+in
+{
+  inherit paths names;
+}

ci/eval/chunk.nix

@@ -1,5 +1,4 @@
-# This file works in tandem with ../../ci/eval/default.nix
-# It turns ./release-outpaths.nix into chunks of a fixed size
+# This turns ./outpaths.nix into chunks of a fixed size.
 {
   lib ? import ../../lib,
   path ? ../..,
@@ -7,7 +6,6 @@
   attrpathFile,
   chunkSize,
   myChunk,
-  checkMeta,
   includeBroken,
   systems,
 }:
@@ -16,9 +14,9 @@ let
   attrpaths = lib.importJSON attrpathFile;
   myAttrpaths = lib.sublist (chunkSize * myChunk) chunkSize attrpaths;
 
-  unfiltered = import ./release-outpaths.nix {
+  unfiltered = import ./outpaths.nix {
     inherit path;
-    inherit checkMeta includeBroken systems;
+    inherit includeBroken systems;
   };
 
   # Turns the unfiltered recursive attribute set into one that is limited to myAttrpaths

ci/eval/compare/default.nix

@@ -105,7 +105,6 @@ let
           ) rebuildsByKernel
           # Set the "11.by: package-maintainer" label to whether all packages directly
           # changed are maintained by the PR's author.
-          # (https://github.com/NixOS/ofborg/blob/df400f44502d4a4a80fa283d33f2e55a4e43ee90/ofborg/src/tagger.rs#L83-L88)
           // {
             "11.by: package-maintainer" =
               maintainers ? ${githubAuthorId}

ci/eval/compare/maintainers.nix

@@ -1,7 +1,6 @@
 {
   lib,
 }:
-# Almost directly vendored from https://github.com/NixOS/ofborg/blob/5a4e743f192fb151915fcbe8789922fa401ecf48/ofborg/src/maintainers.nix
 {
   changedattrs,
   changedpathsjson,

ci/eval/default.nix

@@ -26,6 +26,11 @@ let
       root = ../..;
       fileset = unions (
         map (lib.path.append ../..) [
+          ".version"
+          "ci/supportedSystems.json"
+          "ci/eval/attrpaths.nix"
+          "ci/eval/chunk.nix"
+          "ci/eval/outpaths.nix"
           "default.nix"
           "doc"
           "lib"
@@ -33,8 +38,6 @@ let
           "modules"
           "nixos"
           "pkgs"
-          ".version"
-          "ci/supportedSystems.json"
         ]
       );
     };
@@ -60,7 +63,7 @@ let
         export GC_INITIAL_HEAP_SIZE=4g
         command time -f "Attribute eval done [%MKB max resident, %Es elapsed] %C" \
           nix-instantiate --eval --strict --json --show-trace \
-            "$src/pkgs/top-level/release-attrpaths-superset.nix" \
+            "$src/ci/eval/attrpaths.nix" \
             -A paths \
             -I "$src" \
             --option restrict-eval true \
@@ -78,7 +81,6 @@ let
       attrpathFile ? "${attrpathsSuperset { inherit evalSystem; }}/paths.json",
       # The number of attributes per chunk, see ./README.md for more info.
       chunkSize ? 5000,
-      checkMeta ? true,
 
       # Don't try to eval packages marked as broken.
       includeBroken ? false,
@@ -99,7 +101,7 @@ let
         set +e
         command time -o "$outputDir/timestats/$myChunk" \
           -f "Chunk $myChunk on $system done [%MKB max resident, %Es elapsed] %C" \
-          nix-env -f "${nixpkgs}/pkgs/top-level/release-outpaths-parallel.nix" \
+          nix-env -f "${nixpkgs}/ci/eval/chunk.nix" \
           --eval-system "$system" \
           --option restrict-eval true \
           --option allow-import-from-derivation false \
@@ -110,7 +112,6 @@ let
           --arg myChunk "$myChunk" \
           --arg attrpathFile "${attrpathFile}" \
           --arg systems "[ \"$system\" ]" \
-          --arg checkMeta ${lib.boolToString checkMeta} \
           --arg includeBroken ${lib.boolToString includeBroken} \
           -I ${nixpkgs} \
           -I ${attrpathFile} \

ci/eval/outpaths.nix

@@ -1,19 +1,16 @@
 #!/usr/bin/env nix-shell
 # When using as a callable script, passing `--argstr path some/path` overrides $PWD.
-#!nix-shell -p nix -i "nix-env -qaP --no-name --out-path --arg checkMeta true -f pkgs/top-level/release-outpaths.nix"
+#!nix-shell -p nix -i "nix-env -qaP --no-name --out-path -f ci/eval/outpaths.nix"
 
-# Vendored from:
-#   https://raw.githubusercontent.com/NixOS/ofborg/74f38efa7ef6f0e8e71ec3bfc675ae4fb57d7491/ofborg/src/outpaths.nix
 {
-  checkMeta,
   includeBroken ? true, # set this to false to exclude meta.broken packages from the output
   path ? ./../..,
 
-  # used by pkgs/top-level/release-attrnames-superset.nix
+  # used by ./attrpaths.nix
   attrNamesOnly ? false,
 
   # Set this to `null` to build for builtins.currentSystem only
-  systems ? builtins.fromJSON (builtins.readFile ../../ci/supportedSystems.json),
+  systems ? builtins.fromJSON (builtins.readFile ../supportedSystems.json),
 }:
 let
   lib = import (path + "/lib");
@@ -30,7 +27,7 @@ let
             allowUnfree = true;
             allowInsecurePredicate = x: true;
             allowVariants = !attrNamesOnly;
-            checkMeta = checkMeta;
+            checkMeta = true;
 
             handleEvalIssue =
               reason: errormsg:

pkgs/test/default.nix

@@ -152,9 +152,7 @@ with pkgs;
 
   php = recurseIntoAttrs (callPackages ./php { });
 
-  pkg-config = recurseIntoAttrs (callPackage ../top-level/pkg-config/tests.nix { }) // {
-    __recurseIntoDerivationForReleaseJobs = true;
-  };
+  pkg-config = recurseIntoAttrs (callPackage ../top-level/pkg-config/tests.nix { });
 
   buildRustCrate = recurseIntoAttrs (callPackage ../build-support/rust/build-rust-crate/test { });
   importCargoLock = recurseIntoAttrs (callPackage ../build-support/rust/test/import-cargo-lock { });

pkgs/top-level/all-packages.nix

@@ -5165,10 +5165,7 @@ with pkgs;
           haskell.packages.ghc96
         else
           haskell.packages.ghc98
-      )
-    // {
-      __recurseIntoDerivationForReleaseJobs = true;
-    };
+      );
 
   # haskellPackages.ghc is build->host (it exposes the compiler used to build the
   # set, similarly to stdenv.cc), but pkgs.ghc should be host->target to be more
@@ -8849,22 +8846,18 @@ with pkgs;
     }
   );
 
-  libsForQt5 =
-    (recurseIntoAttrs (
-      import ./qt5-packages.nix {
-        inherit
-          lib
-          config
-          __splicedPackages
-          makeScopeWithSplicing'
-          generateSplicesForMkScope
-          pkgsHostTarget
-          ;
-      }
-    ))
-    // {
-      __recurseIntoDerivationForReleaseJobs = true;
-    };
+  libsForQt5 = recurseIntoAttrs (
+    import ./qt5-packages.nix {
+      inherit
+        lib
+        config
+        __splicedPackages
+        makeScopeWithSplicing'
+        generateSplicesForMkScope
+        pkgsHostTarget
+        ;
+    }
+  );
 
   # plasma5Packages maps to the Qt5 packages set that is used to build the plasma5 desktop
   plasma5Packages = libsForQt5;

pkgs/top-level/linux-kernels.nix

@@ -781,7 +781,6 @@ in
       linux_libre = recurseIntoAttrs (packagesFor kernels.linux_libre);
 
       linux_latest_libre = recurseIntoAttrs (packagesFor kernels.linux_latest_libre);
-      __recurseIntoDerivationForReleaseJobs = true;
     }
     // lib.optionalAttrs config.allowAliases {
 

pkgs/top-level/pkg-config/tests.nix

@@ -22,7 +22,5 @@ let
   };
 in
 lib.recurseIntoAttrs {
-  defaultPkgConfigPackages = allPkgs.callPackage ./test-defaultPkgConfigPackages.nix { } // {
-    __recurseIntoDerivationForReleaseJobs = true;
-  };
+  defaultPkgConfigPackages = allPkgs.callPackage ./test-defaultPkgConfigPackages.nix { };
 }

pkgs/top-level/release-attrpaths-superset.nix

@@ -1,114 +0,0 @@
-# This expression will, as efficiently as possible, dump a
-# *superset* of all attrpaths of derivations which might be
-# part of a release on *any* platform.
-#
-# Both this expression and what ofborg uses (release-outpaths.nix)
-# are essentially single-threaded (under the current cppnix
-# implementation).
-#
-# This expression runs much, much, much faster and uses much, much
-# less memory than the ofborg script by skipping the
-# platform-relevance checks.  The ofborg outpaths.nix script takes
-# half an hour on a 3ghz core and peaks at 60gbytes of memory; this
-# expression runs on the same machine in 44 seconds with peak memory
-# usage of 5gbytes.
-#
-# Once you have the list of attrnames you can split it up into
-# $NUM_CORES batches and run the platform checks separately for each
-# batch, in parallel.
-#
-# To dump the attrnames:
-#
-#   nix-instantiate --eval --strict --json pkgs/top-level/release-attrpaths-superset.nix -A names
-#
-{
-  lib ? import (path + "/lib"),
-  trace ? false,
-  checkMeta ? true,
-  path ? ./../..,
-}:
-let
-
-  # The intended semantics are that an attrpath rooted at pkgs is
-  # part of the (unfiltered) release jobset iff both of the following
-  # are true:
-  #
-  # 1. The attrpath leads to a value for which lib.isDerivation is true
-  #
-  # 2. Any proper prefix of the attrpath at which lib.isDerivation
-  #    is true also has __recurseIntoDerivationForReleaseJobs=true.
-  #
-  # The second condition is unfortunately necessary because there are
-  # Hydra release jobnames which have proper prefixes which are
-  # attrnames of derivations (!).  We should probably restructure
-  # the job tree so that this is not the case.
-  #
-  justAttrNames =
-    path: value:
-    let
-      result =
-        if path == [ "AAAAAASomeThingsFailToEvaluate" ] then
-          [ ]
-        else if
-          lib.isDerivation value
-          &&
-            # in some places we have *derivations* with jobsets as subattributes, ugh
-            !(value.__recurseIntoDerivationForReleaseJobs or false)
-        then
-          [ path ]
-
-        # Even wackier case: we have meta.broken==true jobs with
-        # !meta.broken jobs as subattributes with license=unfree, and
-        # check-meta.nix won't throw an "unfree" failure because the
-        # enclosing derivation is marked broken.  Yeah.  Bonkers.
-        # We should just forbid jobsets enclosed by derivations.
-        else if lib.isDerivation value && !value.meta.available then
-          [ ]
-
-        else if !(lib.isAttrs value) then
-          [ ]
-        else
-          lib.pipe value [
-            (builtins.mapAttrs (
-              name: value:
-              builtins.addErrorContext "while evaluating package set attribute path '${
-                lib.showAttrPath (path ++ [ name ])
-              }'" (justAttrNames (path ++ [ name ]) value)
-            ))
-            builtins.attrValues
-            builtins.concatLists
-          ];
-    in
-    if !trace then result else lib.trace "** ${lib.concatStringsSep "." path}" result;
-
-  releaseOutpaths = import ./release-outpaths.nix {
-    inherit checkMeta;
-    attrNamesOnly = true;
-    inherit path;
-  };
-
-  paths = [
-    # I am not entirely sure why these three packages end up in
-    # the Hydra jobset.  But they do, and they don't meet the
-    # criteria above, so at the moment they are special-cased.
-    [
-      "pkgsLLVM"
-      "stdenv"
-    ]
-    [
-      "pkgsStatic"
-      "stdenv"
-    ]
-    [
-      "pkgsMusl"
-      "stdenv"
-    ]
-  ]
-  ++ justAttrNames [ ] releaseOutpaths;
-
-  names = map (path: (lib.concatStringsSep "." path)) paths;
-
-in
-{
-  inherit paths names;
-}

pkgs/top-level/release-python.nix

@@ -36,11 +36,7 @@ let
       res = builtins.tryEval (
         if isDerivation value then
           value.meta.isBuildPythonPackage or [ ]
-        else if
-          value.recurseForDerivations or false
-          || value.recurseForRelease or false
-          || value.__recurseIntoDerivationForReleaseJobs or false
-        then
+        else if value.recurseForDerivations or false || value.recurseForRelease or false then
           packagePython value
         else
           [ ]

pkgs/top-level/release-unfree-redistributable.nix

@@ -69,11 +69,7 @@ let
               value.meta.hydraPlatforms
                 or (lib.subtractLists (value.meta.badPlatforms or [ ]) (value.meta.platforms or [ "x86_64-linux" ]))
             )
-          else if
-            value.recurseForDerivations or false
-            || value.recurseForRelease or false
-            || value.__recurseIntoDerivationForReleaseJobs or false
-          then
+          else if value.recurseForDerivations or false || value.recurseForRelease or false then
             # Recurse
             packagesWith attrPath cond value
           else

pkgs/top-level/release.nix

@@ -57,12 +57,10 @@
   # resulting tree of attributes to *not* have a ".${system}"
   # suffixed upon every job name like Hydra expects.
   #
-  # This flag exists mainly for use by
-  # pkgs/top-level/release-attrnames-superset.nix; see that file for
-  # full details.  The exact behavior of this flag may change; it
-  # should be considered an internal implementation detail of
-  # pkgs/top-level/.
-  #
+  # This flag exists mainly for use by ci/eval/attrpaths.nix; see
+  # that file for full details.  The exact behavior of this flag
+  # may change; it should be considered an internal implementation
+  # detail of ci/eval.
   attrNamesOnly ? false,
 }: