testers.modularServiceCompliance: init

Portable test suite for modular services, designed to be usable with
other service managers and use cases, inside Nixpkgs and externally.

Like modular services itself, it's still quite bare-bones. New
features can crystallize around this kind of infrastructure.

Assisted-By: Claude Opus 4.7/4.8 era Claude models

doc/build-helpers/testers.chapter.md

@@ -729,4 +729,102 @@ Notable attributes:
 
  * `driverInteractive`: a script that launches an interactive Python session in the context of the `testScript`.
 
+## `modularServiceCompliance` {#tester-modularServiceCompliance}
+
+Compliance suite for [modular service](https://nixos.org/manual/nixos/unstable/#modular-services) integrations.
+
+Tests that a service manager integration correctly handles the portable modular services contract: `process.argv`, sub-services, assertions, and warnings.
+
+### Return value {#tester-modularServiceCompliance-return}
+
+An attribute set of derivations which perform the tests during their build.
+
+### Inputs {#tester-modularServiceCompliance-inputs}
+
+`evalConfig` (function)
+
+: `{ services } -> { config; checkDrv; }`.
+  Function to evaluate the given services in the integration's full context.
+  This function is called for evaluation checks on configurations that will not be run.
+  - Input `services` is an attrset of modular service configurations. These should be used verbatim.
+  - Output attribute `config` is the resulting evaluated services attrset (e.g., the value of the `system.services` option in NixOS).
+    This attribute must be available even if `checkDrv` would fail.
+  - Output attribute `checkDrv` is a representative derivation whose existence and buildability prove the eval is sound (e.g., `system.build.toplevel` in NixOS, but could perhaps be more specific in the case of another process manager integration).
+
+`mkTest` (function)
+
+: `{ name, services, testExe } -> derivation`.
+  - Input `name` is a test name, suitable for use as a derivation name.
+  - Input `services` is an attrset of modular service configurations, matching the structure of the integration's services option.
+  - Input `testExe` is a store path to an executable that verifies the services.
+  - Output: a derivation that runs the service manager with the provided configuration inputs and then calls `testExe` after starting the services. That executable must have access to `sharedDir`.
+
+`sharedDir` (string)
+
+: Path to a directory writable by service processes and readable by `testExe`.
+  The integration must ensure this directory is available when the services and `testExe` run.
+
+:::{.example #ex-modularServiceCompliance-nixos}
+
+# NixOS invocation of the compliance suite
+
+```nix
+# In nixos/tests/all-tests.nix:
+# modularServiceCompliance =
+recurseIntoAttrs (
+  pkgs.testers.modularServiceCompliance {
+    sharedDir = "/tmp/modular-service-compliance";
+    evalConfig =
+      { services }:
+      let
+        machine = evalSystem (
+          { ... }:
+          {
+            system.services = services;
+            system.stateVersion = "25.05";
+            fileSystems."/" = {
+              device = "/test/dummy";
+              fsType = "auto";
+            };
+            boot.loader.grub.enable = false;
+          }
+        );
+      in
+      {
+        config = machine.config.system.services;
+        checkDrv = machine.config.system.build.toplevel;
+      };
+    mkTest =
+      {
+        name,
+        services,
+        testExe,
+      }:
+      runTest {
+        _class = "nixosTest";
+        inherit name;
+        nodes.machine.system.services = services;
+        testScript = ''
+          machine.wait_for_unit("multi-user.target")
+          machine.succeed("${testExe}")
+        '';
+      };
+  }
+)
+```
+
+:::
+
+### Manual compliance items {#tester-modularServiceCompliance-manual}
+
+The following compliance items are not yet automated and must be verified manually when implementing a new modular service integration.
+
+- **Failing assertions prevent deployment.**
+  A service with `assertions = [{ assertion = false; message = "..."; }]` must cause the deployment to fail.
+  The mechanism is integration-specific (e.g., NixOS checks assertions during `system.build.toplevel` evaluation).
+
+- **Warnings are visible to the user.**
+  A service with `warnings = [ "..." ]` must surface the warning to the user.
+  On NixOS these are `builtins.warn` messages emitted during evaluation.
+
 [file system object]: https://nix.dev/manual/nix/latest/store/file-system-object

doc/redirects.json

@@ -105,6 +105,9 @@
   "ex-build-helpers-extendMkDerivation": [
     "index.html#ex-build-helpers-extendMkDerivation"
   ],
+  "ex-modularServiceCompliance-nixos": [
+    "index.html#ex-modularServiceCompliance-nixos"
+  ],
   "ex-pkgs-replace-vars": [
     "index.html#ex-pkgs-replace-vars",
     "index.html#ex-pkgs-substituteAll",
@@ -845,6 +848,18 @@
   "glibcxxassertions": [
     "index.html#glibcxxassertions"
   ],
+  "tester-modularServiceCompliance": [
+    "index.html#tester-modularServiceCompliance"
+  ],
+  "tester-modularServiceCompliance-inputs": [
+    "index.html#tester-modularServiceCompliance-inputs"
+  ],
+  "tester-modularServiceCompliance-manual": [
+    "index.html#tester-modularServiceCompliance-manual"
+  ],
+  "tester-modularServiceCompliance-return": [
+    "index.html#tester-modularServiceCompliance-return"
+  ],
   "tester-shfmt": [
     "index.html#tester-shfmt"
   ],

nixos/tests/all-tests.nix

@@ -1581,6 +1581,16 @@ in
   syncthing-relay = runTest ./syncthing/relay.nix;
   sysfs = runTest ./sysfs.nix;
   sysinit-reactivation = runTest ./sysinit-reactivation.nix;
+  system-services-compliance = recurseIntoAttrs (
+    import ./system-services-compliance.nix {
+      inherit
+        pkgs
+        evalSystem
+        runTest
+        callTest
+        ;
+    }
+  );
   systemd = runTest ./systemd.nix;
   systemd-analyze = runTest ./systemd-analyze.nix;
   systemd-binfmt = handleTestOn [ "x86_64-linux" ] ./systemd-binfmt.nix { };

nixos/tests/system-services-compliance.nix

@@ -0,0 +1,67 @@
+{
+  pkgs,
+  evalSystem,
+  runTest,
+  callTest,
+}:
+
+let
+  sharedDir = "/tmp/modular-service-compliance";
+in
+let
+  suite = pkgs.testers.modularServiceCompliance {
+    inherit sharedDir;
+    namePrefix = "system-services-compliance";
+    evalConfig =
+      { services }:
+      let
+        machine = evalSystem (
+          { ... }:
+          {
+            system.services = services;
+            system.stateVersion = "25.05";
+            fileSystems."/" = {
+              device = "/test/dummy";
+              fsType = "auto";
+            };
+            boot.loader.grub.enable = false;
+          }
+        );
+      in
+      {
+        config = machine.config.system.services;
+        checkDrv = machine.config.system.build.toplevel;
+      };
+    mkTest =
+      {
+        name,
+        services,
+        testExe,
+      }:
+      runTest {
+        _class = "nixosTest";
+        inherit name;
+        nodes.machine.system.services = services;
+        testScript = ''
+          machine.wait_for_unit("multi-user.target")
+          machine.succeed("${testExe}")
+        '';
+        meta.maintainers = with pkgs.lib.maintainers; [ roberth ];
+      };
+  };
+in
+
+# Please the callTest pattern.
+#
+# runTest results already go through findTests/callTest.
+# For plain derivations like `eval`, we apply callTest directly.
+pkgs.lib.mapAttrs (
+  _: v:
+  if v ? test then
+    v
+  else
+    callTest {
+      test = v;
+      driver = v;
+    }
+) suite

pkgs/build-support/testers/default.nix

@@ -260,4 +260,6 @@
   shellcheck = callPackage ./shellcheck/tester.nix { };
 
   shfmt = callPackage ./shfmt { };
+
+  modularServiceCompliance = callPackage ./modular-service-compliance.nix { };
 }

pkgs/build-support/testers/modular-service-compliance.nix

@@ -0,0 +1,251 @@
+{
+  lib,
+  coreutils,
+  gnugrep,
+  writeShellScript,
+  writeShellApplication,
+  stdenvNoCC,
+}:
+
+/**
+  See https://nixos.org/manual/nixpkgs/unstable/#tester-modularServiceCompliance
+  or doc/build-helpers/testers.chapter.md
+*/
+{
+  evalConfig,
+  mkTest,
+  sharedDir,
+  namePrefix ? "modular-service-compliance",
+}:
+
+let
+
+  /**
+    A successful evaluation to be probed by evalTestDefs
+  */
+  # Try use only few evalConfig calls for performance.
+  evalResult = evalConfig {
+    services = {
+      svc = {
+        process.argv = [ "${coreutils}/bin/true" ];
+        assertions = [
+          {
+            assertion = true;
+            message = "compliance test assertion";
+          }
+        ];
+        warnings = [ "compliance test warning" ];
+        services.child = {
+          process.argv = [ "${coreutils}/bin/true" ];
+          assertions = [
+            {
+              assertion = true;
+              message = "compliance child assertion";
+            }
+          ];
+          warnings = [ "compliance child warning" ];
+        };
+      };
+    };
+  };
+
+  evalTestDefs =
+    let
+      c = evalResult.config.svc;
+    in
+    {
+      testProcessArgv = {
+        expr = c.process.argv;
+        expected = [ "${coreutils}/bin/true" ];
+      };
+
+      testSubServiceArgv = {
+        expr = c.services.child.process.argv;
+        expected = [ "${coreutils}/bin/true" ];
+      };
+
+      testAssertions = {
+        expr = builtins.elem {
+          assertion = true;
+          message = "compliance test assertion";
+        } c.assertions;
+        expected = true;
+      };
+
+      testWarnings = {
+        expr = builtins.elem "compliance test warning" c.warnings;
+        expected = true;
+      };
+
+      testSubServiceAssertions = {
+        expr = builtins.elem {
+          assertion = true;
+          message = "compliance child assertion";
+        } c.services.child.assertions;
+        expected = true;
+      };
+
+      testSubServiceWarnings = {
+        expr = builtins.elem "compliance child warning" c.services.child.warnings;
+        expected = true;
+      };
+
+      # Separate eval for a failing assertion — checkDrv would fail here,
+      # so we only access config.
+      testFailingAssertionValue = {
+        expr = builtins.elem {
+          assertion = false;
+          message = "compliance failing assertion";
+        } failingEval.config.failing.assertions;
+        expected = true;
+      };
+    };
+
+  failingEval = evalConfig {
+    services = {
+      failing = {
+        process.argv = [ "${coreutils}/bin/true" ];
+        assertions = [
+          {
+            assertion = false;
+            message = "compliance failing assertion";
+          }
+        ];
+      };
+    };
+  };
+
+  /**
+    A service script that records its received arguments, then sleeps forever.
+    The first argument is a service identifier used to namespace its comms
+    subdirectory; the remaining arguments are recorded as the service's args.
+  */
+  svc = writeShellScript "${namePrefix}-svc" ''
+    id="$1"; shift
+    dir="${sharedDir}/$id"
+    mkdir -p "$dir"
+    echo "$$" > "$dir/pid"
+    printf '%s\n' "$@" > "$dir/args"
+    exec "${coreutils}/bin/sleep" infinity
+  '';
+
+  mkArgv =
+    id: extraArgs:
+    [
+      svc
+      id
+    ]
+    ++ extraArgs;
+
+  /**
+    Shell snippet: wait for a service to write its pid file, verify the
+    process is still alive, then check that each expected argument appears
+    in the recorded args file.
+  */
+  waitAndCheck =
+    id: expectedArgs:
+    ''
+      echo "waiting for ${id}..."
+      timeout=30; elapsed=0
+      while [ ! -f "${sharedDir}/${id}/pid" ] && [ "$elapsed" -lt "$timeout" ]; do
+        sleep 1; elapsed=$((elapsed + 1))
+      done
+      test -f "${sharedDir}/${id}/pid" || { echo "${id}: no pid file after ''${timeout}s"; exit 1; }
+      pid=$(cat "${sharedDir}/${id}/pid")
+      kill -0 "$pid" || { echo "${id}: pid $pid is not running"; exit 1; }
+      echo "${id}: started (pid $pid)"
+    ''
+    + lib.concatMapStrings (arg: ''
+      grep -qxF -- ${lib.escapeShellArg arg} "${sharedDir}/${id}/args" \
+        || { echo "${id}: expected arg ${lib.escapeShellArg arg} not found"; cat "${sharedDir}/${id}/args"; exit 1; }
+    '') expectedArgs;
+
+  mkTestScript =
+    name: text:
+    lib.getExe (writeShellApplication {
+      name = "${namePrefix}-${name}";
+      runtimeInputs = [
+        coreutils
+        gnugrep
+      ];
+      inherit text;
+    });
+
+in
+{
+  # Eval-level tests: config structure, evaluated in the integration's
+  # full context (one whole-system eval).
+  # TODO: generalize with
+  #   - pkgs/test/buildenv.nix
+  #   - pkgs/test/overriding.nix
+  eval = stdenvNoCC.mkDerivation (finalAttrs: {
+    __structuredAttrs = true;
+    name = "${namePrefix}-eval-report";
+    # Depend on the integration's representative derivation to prove that
+    # the system builds with these services.
+    representative = evalResult.checkDrv;
+    passthru = {
+      tests = evalTestDefs;
+      failures = lib.runTests finalAttrs.passthru.tests;
+    };
+    testResults = lib.mapAttrs (_: test: test.expr == test.expected) finalAttrs.passthru.tests;
+    buildCommand = ''
+      touch $out
+      for testName in "''${!testResults[@]}"; do
+        if [[ -n "''${testResults[$testName]}" ]]; then
+          echo "PASS  $testName"
+        else
+          echo "FAIL  $testName"
+        fi
+      done
+    ''
+    + lib.optionalString (lib.any (v: !v) (lib.attrValues finalAttrs.testResults)) ''
+      {
+        echo ""
+        echo "Eval-level compliance failures:"
+        for testName in "''${!testResults[@]}"; do
+          if [[ -z "''${testResults[$testName]}" ]]; then
+            echo "- $testName"
+          fi
+        done
+        echo ""
+        echo 'Inspect with: nix eval .#<path>.tests.''${testName}'
+      } >&2
+      exit 1
+    '';
+  });
+
+  # Integration tests: verify that services actually run.
+
+  basic-argv = mkTest {
+    name = "${namePrefix}-basic-argv";
+    services.test.process.argv = mkArgv "test" [
+      "--greeting"
+      "hello"
+    ];
+    testExe = mkTestScript "basic-argv" (
+      waitAndCheck "test" [
+        "--greeting"
+        "hello"
+      ]
+    );
+  };
+
+  sub-services = mkTest {
+    name = "${namePrefix}-sub-services";
+    services.a = {
+      process.argv = mkArgv "a" [ "--depth=0" ];
+      services.b = {
+        process.argv = mkArgv "b" [ "--depth=1" ];
+        services.c.process.argv = mkArgv "c" [ "--depth=2" ];
+      };
+    };
+    testExe = mkTestScript "sub-services" ''
+      ${waitAndCheck "a" [ "--depth=0" ]}
+      ${waitAndCheck "b" [ "--depth=1" ]}
+      ${waitAndCheck "c" [ "--depth=2" ]}
+    '';
+  };
+
+  # See also the manual compliance items in doc/build-helpers/testers.chapter.md.
+}