diff --git a/doc/build-helpers/testers.chapter.md b/doc/build-helpers/testers.chapter.md index 632b11ab12ea..2c02f1947c3f 100644 --- a/doc/build-helpers/testers.chapter.md +++ b/doc/build-helpers/testers.chapter.md @@ -729,4 +729,102 @@ Notable attributes: * `driverInteractive`: a script that launches an interactive Python session in the context of the `testScript`. +## `modularServiceCompliance` {#tester-modularServiceCompliance} + +Compliance suite for [modular service](https://nixos.org/manual/nixos/unstable/#modular-services) integrations. + +Tests that a service manager integration correctly handles the portable modular services contract: `process.argv`, sub-services, assertions, and warnings. + +### Return value {#tester-modularServiceCompliance-return} + +An attribute set of derivations which perform the tests during their build. + +### Inputs {#tester-modularServiceCompliance-inputs} + +`evalConfig` (function) + +: `{ services } -> { config; checkDrv; }`. + Function to evaluate the given services in the integration's full context. + This function is called for evaluation checks on configurations that will not be run. + - Input `services` is an attrset of modular service configurations. These should be used verbatim. + - Output attribute `config` is the resulting evaluated services attrset (e.g., the value of the `system.services` option in NixOS). + This attribute must be available even if `checkDrv` would fail. + - Output attribute `checkDrv` is a representative derivation whose existence and buildability prove the eval is sound (e.g., `system.build.toplevel` in NixOS, but could perhaps be more specific in the case of another process manager integration). + +`mkTest` (function) + +: `{ name, services, testExe } -> derivation`. + - Input `name` is a test name, suitable for use as a derivation name. + - Input `services` is an attrset of modular service configurations, matching the structure of the integration's services option. + - Input `testExe` is a store path to an executable that verifies the services. + - Output: a derivation that runs the service manager with the provided configuration inputs and then calls `testExe` after starting the services. That executable must have access to `sharedDir`. + +`sharedDir` (string) + +: Path to a directory writable by service processes and readable by `testExe`. + The integration must ensure this directory is available when the services and `testExe` run. + +:::{.example #ex-modularServiceCompliance-nixos} + +# NixOS invocation of the compliance suite + +```nix +# In nixos/tests/all-tests.nix: +# modularServiceCompliance = +recurseIntoAttrs ( + pkgs.testers.modularServiceCompliance { + sharedDir = "/tmp/modular-service-compliance"; + evalConfig = + { services }: + let + machine = evalSystem ( + { ... }: + { + system.services = services; + system.stateVersion = "25.05"; + fileSystems."/" = { + device = "/test/dummy"; + fsType = "auto"; + }; + boot.loader.grub.enable = false; + } + ); + in + { + config = machine.config.system.services; + checkDrv = machine.config.system.build.toplevel; + }; + mkTest = + { + name, + services, + testExe, + }: + runTest { + _class = "nixosTest"; + inherit name; + nodes.machine.system.services = services; + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed("${testExe}") + ''; + }; + } +) +``` + +::: + +### Manual compliance items {#tester-modularServiceCompliance-manual} + +The following compliance items are not yet automated and must be verified manually when implementing a new modular service integration. + +- **Failing assertions prevent deployment.** + A service with `assertions = [{ assertion = false; message = "..."; }]` must cause the deployment to fail. + The mechanism is integration-specific (e.g., NixOS checks assertions during `system.build.toplevel` evaluation). + +- **Warnings are visible to the user.** + A service with `warnings = [ "..." ]` must surface the warning to the user. + On NixOS these are `builtins.warn` messages emitted during evaluation. + [file system object]: https://nix.dev/manual/nix/latest/store/file-system-object diff --git a/doc/redirects.json b/doc/redirects.json index 646b12a5a077..488ad7c6fab0 100644 --- a/doc/redirects.json +++ b/doc/redirects.json @@ -105,6 +105,9 @@ "ex-build-helpers-extendMkDerivation": [ "index.html#ex-build-helpers-extendMkDerivation" ], + "ex-modularServiceCompliance-nixos": [ + "index.html#ex-modularServiceCompliance-nixos" + ], "ex-pkgs-replace-vars": [ "index.html#ex-pkgs-replace-vars", "index.html#ex-pkgs-substituteAll", @@ -845,6 +848,18 @@ "glibcxxassertions": [ "index.html#glibcxxassertions" ], + "tester-modularServiceCompliance": [ + "index.html#tester-modularServiceCompliance" + ], + "tester-modularServiceCompliance-inputs": [ + "index.html#tester-modularServiceCompliance-inputs" + ], + "tester-modularServiceCompliance-manual": [ + "index.html#tester-modularServiceCompliance-manual" + ], + "tester-modularServiceCompliance-return": [ + "index.html#tester-modularServiceCompliance-return" + ], "tester-shfmt": [ "index.html#tester-shfmt" ], diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index e49f415dc2a1..8949b0483877 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -1581,6 +1581,16 @@ in syncthing-relay = runTest ./syncthing/relay.nix; sysfs = runTest ./sysfs.nix; sysinit-reactivation = runTest ./sysinit-reactivation.nix; + system-services-compliance = recurseIntoAttrs ( + import ./system-services-compliance.nix { + inherit + pkgs + evalSystem + runTest + callTest + ; + } + ); systemd = runTest ./systemd.nix; systemd-analyze = runTest ./systemd-analyze.nix; systemd-binfmt = handleTestOn [ "x86_64-linux" ] ./systemd-binfmt.nix { }; diff --git a/nixos/tests/system-services-compliance.nix b/nixos/tests/system-services-compliance.nix new file mode 100644 index 000000000000..0a40d18fc4fe --- /dev/null +++ b/nixos/tests/system-services-compliance.nix @@ -0,0 +1,67 @@ +{ + pkgs, + evalSystem, + runTest, + callTest, +}: + +let + sharedDir = "/tmp/modular-service-compliance"; +in +let + suite = pkgs.testers.modularServiceCompliance { + inherit sharedDir; + namePrefix = "system-services-compliance"; + evalConfig = + { services }: + let + machine = evalSystem ( + { ... }: + { + system.services = services; + system.stateVersion = "25.05"; + fileSystems."/" = { + device = "/test/dummy"; + fsType = "auto"; + }; + boot.loader.grub.enable = false; + } + ); + in + { + config = machine.config.system.services; + checkDrv = machine.config.system.build.toplevel; + }; + mkTest = + { + name, + services, + testExe, + }: + runTest { + _class = "nixosTest"; + inherit name; + nodes.machine.system.services = services; + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed("${testExe}") + ''; + meta.maintainers = with pkgs.lib.maintainers; [ roberth ]; + }; + }; +in + +# Please the callTest pattern. +# +# runTest results already go through findTests/callTest. +# For plain derivations like `eval`, we apply callTest directly. +pkgs.lib.mapAttrs ( + _: v: + if v ? test then + v + else + callTest { + test = v; + driver = v; + } +) suite diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index 4dca26274747..5d9f568d2ed3 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -260,4 +260,6 @@ shellcheck = callPackage ./shellcheck/tester.nix { }; shfmt = callPackage ./shfmt { }; + + modularServiceCompliance = callPackage ./modular-service-compliance.nix { }; } diff --git a/pkgs/build-support/testers/modular-service-compliance.nix b/pkgs/build-support/testers/modular-service-compliance.nix new file mode 100644 index 000000000000..ece674583eff --- /dev/null +++ b/pkgs/build-support/testers/modular-service-compliance.nix @@ -0,0 +1,251 @@ +{ + lib, + coreutils, + gnugrep, + writeShellScript, + writeShellApplication, + stdenvNoCC, +}: + +/** + See https://nixos.org/manual/nixpkgs/unstable/#tester-modularServiceCompliance + or doc/build-helpers/testers.chapter.md +*/ +{ + evalConfig, + mkTest, + sharedDir, + namePrefix ? "modular-service-compliance", +}: + +let + + /** + A successful evaluation to be probed by evalTestDefs + */ + # Try use only few evalConfig calls for performance. + evalResult = evalConfig { + services = { + svc = { + process.argv = [ "${coreutils}/bin/true" ]; + assertions = [ + { + assertion = true; + message = "compliance test assertion"; + } + ]; + warnings = [ "compliance test warning" ]; + services.child = { + process.argv = [ "${coreutils}/bin/true" ]; + assertions = [ + { + assertion = true; + message = "compliance child assertion"; + } + ]; + warnings = [ "compliance child warning" ]; + }; + }; + }; + }; + + evalTestDefs = + let + c = evalResult.config.svc; + in + { + testProcessArgv = { + expr = c.process.argv; + expected = [ "${coreutils}/bin/true" ]; + }; + + testSubServiceArgv = { + expr = c.services.child.process.argv; + expected = [ "${coreutils}/bin/true" ]; + }; + + testAssertions = { + expr = builtins.elem { + assertion = true; + message = "compliance test assertion"; + } c.assertions; + expected = true; + }; + + testWarnings = { + expr = builtins.elem "compliance test warning" c.warnings; + expected = true; + }; + + testSubServiceAssertions = { + expr = builtins.elem { + assertion = true; + message = "compliance child assertion"; + } c.services.child.assertions; + expected = true; + }; + + testSubServiceWarnings = { + expr = builtins.elem "compliance child warning" c.services.child.warnings; + expected = true; + }; + + # Separate eval for a failing assertion — checkDrv would fail here, + # so we only access config. + testFailingAssertionValue = { + expr = builtins.elem { + assertion = false; + message = "compliance failing assertion"; + } failingEval.config.failing.assertions; + expected = true; + }; + }; + + failingEval = evalConfig { + services = { + failing = { + process.argv = [ "${coreutils}/bin/true" ]; + assertions = [ + { + assertion = false; + message = "compliance failing assertion"; + } + ]; + }; + }; + }; + + /** + A service script that records its received arguments, then sleeps forever. + The first argument is a service identifier used to namespace its comms + subdirectory; the remaining arguments are recorded as the service's args. + */ + svc = writeShellScript "${namePrefix}-svc" '' + id="$1"; shift + dir="${sharedDir}/$id" + mkdir -p "$dir" + echo "$$" > "$dir/pid" + printf '%s\n' "$@" > "$dir/args" + exec "${coreutils}/bin/sleep" infinity + ''; + + mkArgv = + id: extraArgs: + [ + svc + id + ] + ++ extraArgs; + + /** + Shell snippet: wait for a service to write its pid file, verify the + process is still alive, then check that each expected argument appears + in the recorded args file. + */ + waitAndCheck = + id: expectedArgs: + '' + echo "waiting for ${id}..." + timeout=30; elapsed=0 + while [ ! -f "${sharedDir}/${id}/pid" ] && [ "$elapsed" -lt "$timeout" ]; do + sleep 1; elapsed=$((elapsed + 1)) + done + test -f "${sharedDir}/${id}/pid" || { echo "${id}: no pid file after ''${timeout}s"; exit 1; } + pid=$(cat "${sharedDir}/${id}/pid") + kill -0 "$pid" || { echo "${id}: pid $pid is not running"; exit 1; } + echo "${id}: started (pid $pid)" + '' + + lib.concatMapStrings (arg: '' + grep -qxF -- ${lib.escapeShellArg arg} "${sharedDir}/${id}/args" \ + || { echo "${id}: expected arg ${lib.escapeShellArg arg} not found"; cat "${sharedDir}/${id}/args"; exit 1; } + '') expectedArgs; + + mkTestScript = + name: text: + lib.getExe (writeShellApplication { + name = "${namePrefix}-${name}"; + runtimeInputs = [ + coreutils + gnugrep + ]; + inherit text; + }); + +in +{ + # Eval-level tests: config structure, evaluated in the integration's + # full context (one whole-system eval). + # TODO: generalize with + # - pkgs/test/buildenv.nix + # - pkgs/test/overriding.nix + eval = stdenvNoCC.mkDerivation (finalAttrs: { + __structuredAttrs = true; + name = "${namePrefix}-eval-report"; + # Depend on the integration's representative derivation to prove that + # the system builds with these services. + representative = evalResult.checkDrv; + passthru = { + tests = evalTestDefs; + failures = lib.runTests finalAttrs.passthru.tests; + }; + testResults = lib.mapAttrs (_: test: test.expr == test.expected) finalAttrs.passthru.tests; + buildCommand = '' + touch $out + for testName in "''${!testResults[@]}"; do + if [[ -n "''${testResults[$testName]}" ]]; then + echo "PASS $testName" + else + echo "FAIL $testName" + fi + done + '' + + lib.optionalString (lib.any (v: !v) (lib.attrValues finalAttrs.testResults)) '' + { + echo "" + echo "Eval-level compliance failures:" + for testName in "''${!testResults[@]}"; do + if [[ -z "''${testResults[$testName]}" ]]; then + echo "- $testName" + fi + done + echo "" + echo 'Inspect with: nix eval .#.tests.''${testName}' + } >&2 + exit 1 + ''; + }); + + # Integration tests: verify that services actually run. + + basic-argv = mkTest { + name = "${namePrefix}-basic-argv"; + services.test.process.argv = mkArgv "test" [ + "--greeting" + "hello" + ]; + testExe = mkTestScript "basic-argv" ( + waitAndCheck "test" [ + "--greeting" + "hello" + ] + ); + }; + + sub-services = mkTest { + name = "${namePrefix}-sub-services"; + services.a = { + process.argv = mkArgv "a" [ "--depth=0" ]; + services.b = { + process.argv = mkArgv "b" [ "--depth=1" ]; + services.c.process.argv = mkArgv "c" [ "--depth=2" ]; + }; + }; + testExe = mkTestScript "sub-services" '' + ${waitAndCheck "a" [ "--depth=0" ]} + ${waitAndCheck "b" [ "--depth=1" ]} + ${waitAndCheck "c" [ "--depth=2" ]} + ''; + }; + + # See also the manual compliance items in doc/build-helpers/testers.chapter.md. +}