[Backport release-26.05] lib.sources.sourceByGlobs: init function (#538314)

lib/sources.nix

@@ -7,12 +7,19 @@ let
     match
     split
     storeDir
+    escapeRegex
+    removePrefix
     ;
   inherit (lib)
     boolToString
     filter
     isString
     readFile
+    concatStrings
+    length
+    elemAt
+    isList
+    any
     ;
   inherit (lib.filesystem)
     pathIsRegularFile
@@ -513,6 +520,113 @@ let
     else
       throw "repoRevToName: invalid kind";
 
+  /**
+    Filter a source tree by a list of doublestar-style glob patterns,
+    returning a source that only contains paths matching at least one
+    pattern. `*` matches a single path component, and `**` matches any
+    number of components.
+
+    # Inputs
+
+    `src`
+
+    : The source tree to filter.
+
+    `patterns`
+
+    : List of glob patterns to include, e.g. `[ "*.py" "src/**" ]`.
+      A leading `**` (e.g. `**\/*.py` for all `.py` files at any depth)
+      is also supported; the `\` here is just a Nix string escape used
+      to avoid closing this comment.
+
+    # Examples
+    :::{.example}
+    ## `sourceByGlobs` usage example
+
+    - Include everything under a subdirectory
+    ```nix
+    src = sourceByGlobs ./. [ "src/**" "tests/**" ]
+    ```
+
+    - Include all .py files in root directory only
+    ```nix
+    src = sourceByGlobs ./. [ "*.py" ]
+    ```
+
+    :::
+  */
+  sourceByGlobs =
+    let
+      splitPath = path: filter isString (split "/" path);
+      # Make component regex
+      mkRe =
+        s:
+        if s == "**" then
+          ".*" # Has special handling below
+        else
+          concatStrings (map (tok: if isList tok then "[^/]*" else escapeRegex tok) (split "\\*+" s));
+
+      # Make a source filter function from pattern
+      mkMatcher =
+        pat:
+        let
+          globs = map mkRe (splitPath pat);
+          glen = length globs;
+        in
+        path: type:
+        let
+          path' = splitPath path;
+          plen = length path';
+
+          recurse =
+            gi: pi:
+            let
+              g = elemAt globs gi;
+              p = elemAt path' pi;
+              m = match g p != null;
+            in
+            if pi >= plen then # Reached end of path
+              gi >= glen || (type == "directory" || type == "symlink") # Only allow partial matches for directories
+            else if gi >= glen then # Reached end of globs
+              false
+            else if g == ".*" then # Special handling for **
+              (
+                # Lookahead for next glob match
+                if (gi + 1) == glen then
+                  true
+                else if (match (elemAt globs (gi + 1)) p != null) then
+                  recurse (gi + 1) pi
+                else if m then
+                  recurse gi (pi + 1)
+                else
+                  false
+              )
+            else if m then
+              recurse (gi + 1) (pi + 1)
+            else
+              false;
+
+        in
+        recurse 0 0;
+
+      mkSourceFilter =
+        root: patterns:
+        let
+          root' = "${toString root}/";
+          matchers = map mkMatcher patterns;
+        in
+        name: type:
+        let
+          name' = removePrefix root' name;
+        in
+        any (m: m name' type) matchers;
+
+    in
+    src: patterns:
+    lib.cleanSourceWith {
+      filter = mkSourceFilter src patterns;
+      inherit src;
+    };
 in
 {
   inherit
@@ -532,6 +646,7 @@ in
 
     sourceByRegex
     sourceFilesBySuffices
+    sourceByGlobs
 
     trace
     ;

lib/tests/sources.sh

@@ -70,4 +70,16 @@ dir="$(nix-instantiate --eval --strict --read-write-mode --json --expr '(with im
 EOF
 ) || die "cleanSourceWith + cleanSource"
 
+
+dir="$(nix-instantiate --eval --strict --read-write-mode --json --expr '(with import <nixpkgs/lib>; "${
+  sources.sourceByGlobs '"$work"' [ "*.md" "**/*.o" ]
+}")' | crudeUnquoteJSON)"
+(cd "$dir"; find) | sort -f | diff -U10 - <(cat <<EOF
+.
+./module.o
+./README.md
+EOF
+) || die "sourceByGlobs 1"
+
+
 echo >&2 tests ok

pkgs/applications/editors/vim/plugins/utils/updater.nix

@@ -43,7 +43,7 @@ buildPythonApplication {
         neovim-unwrapped
         nurl
       ]
-    }" --prefix PYTHONPATH : "${./.}" )
+    }" --prefix PYTHONPATH : "${lib.sources.sourceByGlobs ./. [ "**/*.py" ]}" )
     wrapPythonPrograms
   '';
 

pkgs/development/compilers/purescript/purescript/test-minimal-module/default.nix

@@ -1,11 +1,17 @@
 {
+  lib,
   runCommand,
   purescript,
   nodejs,
 }:
 
 runCommand "purescript-test-minimal-module" { } ''
-  ${purescript}/bin/purs compile -o ./output ${./.}/Main.purs
+  ${purescript}/bin/purs compile -o ./output ${
+    lib.sources.sourceByGlobs ./. [
+      "*.purs"
+      "*.js"
+    ]
+  }/Main.purs
 
   echo 'import {main} from "./output/Main/index.js"; main()' > node.mjs
 

pkgs/pkgs-lib/formats/java-properties/test/default.nix

@@ -72,8 +72,8 @@ stdenv.mkDerivation {
     )
   );
 
-  src = lib.sourceByRegex ./. [
-    ".*\\.java"
+  src = lib.sources.sourceByGlobs ./. [
+    "**/*.java"
   ];
   # On Linux, this can be C.UTF-8, but darwin + zulu requires en_US.UTF-8
   LANG = "en_US.UTF-8";