c4patino/nixpkgs
1
0
Fork 0
mirror of https://github.com/NixOS/nixpkgs.git synced 2026-07-06 17:13:24 -05:00
Code Issues Projects Releases Packages Wiki Activity

ci/eval/compare: Improve performance and avoid large stacks

Browse source 
Various improvements such as:
1. Avoiding deduplications when there can't be any duplicates
2. Avoiding O(n^2) deduplications
3. Using builtins.any to avoid list allocations
4. Using builtins.concatMap instead of lib.flatten when it's known that there's only one level of nesting
5. Using builtins.groupBy instead of folding with an accumulator

In particular 5. should fix CI exceeding the stack size on staging: https://github.com/NixOS/nixpkgs/actions/runs/12989244871/job/36240781244?pr=377253

While 2. in particular should make CI a lot faster.
This commit is contained in:
Silvan Mosberger 2025-01-27 22:44:51 +01:00
commit 80e011146b
2 changed files with 16 additions and 32 deletions
Download patch file Download diff file Expand all files Collapse all files

5
ci/eval/compare/default.nix
View file

@ -69,7 +69,6 @@ let
groupByPlatform
extractPackageNames
getLabels
uniqueStrings
;
getAttrs = dir: builtins.fromJSON (builtins.readFile "${dir}/outpaths.json");
@ -81,7 +80,7 @@ let
# - values: lists of `packagePlatformPath`s
diffAttrs = diff beforeAttrs afterAttrs;
rebuilds = uniqueStrings (diffAttrs.added ++ diffAttrs.changed);
rebuilds = diffAttrs.added ++ diffAttrs.changed;
rebuildsPackagePlatformAttrs = convertToPackagePlatformAttrs rebuilds;
changed-paths =
@ -110,7 +109,7 @@ let
);
maintainers = import ./maintainers.nix {
changedattrs = lib.unique (map (a: a.packagePath) rebuildsPackagePlatformAttrs);
changedattrs = lib.attrNames (lib.groupBy (a: a.name) rebuildsPackagePlatformAttrs);
changedpathsjson = touchedFilesJson;
};
in

43
ci/eval/compare/maintainers.nix
View file

@ -11,17 +11,13 @@ let
changedpaths = builtins.fromJSON (builtins.readFile changedpathsjson);
anyMatchingFile =
filename:
let
matching = builtins.filter (changed: lib.strings.hasSuffix changed filename) changedpaths;
in
(builtins.length matching) > 0;
filename: builtins.any (changed: lib.strings.hasSuffix changed filename) changedpaths;
anyMatchingFiles = files: (builtins.length (builtins.filter anyMatchingFile files)) > 0;
anyMatchingFiles = files: builtins.any anyMatchingFile files;
enrichedAttrs = builtins.map (path: {
path = path;
name = builtins.concatStringsSep "." path;
enrichedAttrs = builtins.map (name: {
path = lib.splitString "." name;
name = name;
}) changedattrs;
validPackageAttributes = builtins.filter (
@ -80,27 +76,16 @@ let
attrsWithModifiedFiles = builtins.filter (pkg: anyMatchingFiles pkg.filenames) attrsWithFilenames;
listToPing = lib.lists.flatten (
builtins.map (
pkg:
builtins.map (maintainer: {
id = maintainer.githubId;
packageName = pkg.name;
dueToFiles = pkg.filenames;
}) pkg.maintainers
) attrsWithModifiedFiles
);
byMaintainer = lib.lists.foldr (
ping: collector:
collector
// {
"${toString ping.id}" = [
{ inherit (ping) packageName dueToFiles; }
] ++ (collector."${toString ping.id}" or [ ]);
}
) { } listToPing;
listToPing = lib.concatMap (
pkg:
builtins.map (maintainer: {
id = maintainer.githubId;
packageName = pkg.name;
dueToFiles = pkg.filenames;
}) pkg.maintainers
) attrsWithModifiedFiles;
byMaintainer = lib.groupBy (ping: toString ping.id) listToPing;
packagesPerMaintainer = lib.attrsets.mapAttrs (
maintainer: packages: builtins.map (pkg: pkg.packageName) packages