diff --git a/nixos/doc/manual/development/running-nixos-tests-interactively.section.md b/nixos/doc/manual/development/running-nixos-tests-interactively.section.md index 572173cb4fbf..121abd51c804 100644 --- a/nixos/doc/manual/development/running-nixos-tests-interactively.section.md +++ b/nixos/doc/manual/development/running-nixos-tests-interactively.section.md @@ -88,52 +88,34 @@ An SSH-based backdoor to log into machines can be enabled with } ``` -::: {.warning} -Make sure to only enable the backdoor for interactive tests -(i.e. by using `interactive.sshBackdoor.enable`)! This is the only -supported configuration. - -Running a test in a sandbox with this will fail because `/dev/vhost-vsock` isn't available -in the sandbox. -::: - This creates a [vsock socket](https://man7.org/linux/man-pages/man7/vsock.7.html) for each VM to log in with SSH. This configures root login with an empty password. -When the VMs get started interactively with the test-driver, it's possible to -connect to `machine` with +On the host-side a UNIX domain-socket is used with +[vhost-device-vsock](https://github.com/rust-vmm/vhost-device/blob/main/vhost-device-vsock/README.md). +That way, it's not necessary to assign system-wide unique vsock numbers. ``` -$ ssh vsock/3 -o User=root +$ ssh vsock-mux//tmp/path/to/host -o User=root ``` -The socket numbers correspond to the node number of the test VM, but start -at three instead of one because that's the lowest possible -vsock number. The exact SSH commands are also printed out when starting -`nixos-test-driver`. +The socket paths are printed when starting the test driver: + +``` +Note: this requires systemd-ssh-proxy(1) to be enabled (default on NixOS 25.05 and newer). + machine: ssh -o User=root vsock-mux//tmp/tmpg1rp9nti/machine_host.socket +``` On non-NixOS systems you'll probably need to enable the SSH config from {manpage}`systemd-ssh-proxy(1)` yourself. -If starting VM fails with an error like +During a test-run, it's possible to print the SSH commands again by running ``` -qemu-system-x86_64: -device vhost-vsock-pci,guest-cid=3: vhost-vsock: unable to set guest cid: Address already in use -``` - -it means that the vsock numbers for the VMs are already in use. This can happen -if another interactive test with SSH backdoor enabled is running on the machine. - -In that case, you need to assign another range of vsock numbers. You can pick another -offset with - -```nix -{ - sshBackdoor = { - enable = true; - vsockOffset = 23542; - }; -} +In [2]: dump_machine_ssh() +SSH backdoor enabled, the machines can be accessed like this: +Note: this requires systemd-ssh-proxy(1) to be enabled (default on NixOS 25.05 and newer). + machine: ssh -o User=root vsock-mux//tmp/tmpg1rp9nti/machine_host.socket ``` ## Port forwarding to NixOS test VMs {#sec-nixos-test-port-forwarding} diff --git a/nixos/doc/manual/development/writing-nixos-tests.section.md b/nixos/doc/manual/development/writing-nixos-tests.section.md index 7e7414bd61a9..8296b4904d3a 100644 --- a/nixos/doc/manual/development/writing-nixos-tests.section.md +++ b/nixos/doc/manual/development/writing-nixos-tests.section.md @@ -512,19 +512,11 @@ Once you are in the sandbox shell, you can access the VMs (for example, `machine with SSH over vsock: ``` -bash# ssh -F ./ssh_config vsock/3 +bash# ssh -F ./ssh_config -o User=root vsock-mux//tmp/.../machine_host.socket ``` -For the AF_VSOCK feature to work, `/dev/vhost-vsock` is needed in the sandbox -which can be done with e.g. - -``` -nix-build -A nixosTests.foo --option sandbox-paths /dev/vhost-vsock -``` - -As described in [](#sec-nixos-test-ssh-access), the numbers for vsock start at -`3` instead of `1`. So the first VM in the network (sorted alphabetically) can -be accessed with `vsock/3`. +The socket paths are printed at the beginning of the test. See +[](#sec-nixos-test-ssh-access) for more context. ### SSH access to test containers {#sec-test-container-ssh-access} diff --git a/nixos/doc/manual/redirects.json b/nixos/doc/manual/redirects.json index feb535c2142c..ad1ef882c85f 100644 --- a/nixos/doc/manual/redirects.json +++ b/nixos/doc/manual/redirects.json @@ -2174,9 +2174,6 @@ "test-opt-sshBackdoor.enable": [ "index.html#test-opt-sshBackdoor.enable" ], - "test-opt-sshBackdoor.vsockOffset": [ - "index.html#test-opt-sshBackdoor.vsockOffset" - ], "test-opt-enableDebugHook": [ "index.html#test-opt-enableDebugHook" ], diff --git a/nixos/lib/test-driver/default.nix b/nixos/lib/test-driver/default.nix index 72aa6b11bfe5..65ddb4049b0c 100644 --- a/nixos/lib/test-driver/default.nix +++ b/nixos/lib/test-driver/default.nix @@ -14,6 +14,7 @@ remote-pdb, netpbm, + vhost-device-vsock, nixosTests, qemu_pkg ? qemu_test, qemu_test, @@ -56,6 +57,7 @@ buildPythonApplication { socat util-linux vde2 + vhost-device-vsock ] ++ lib.optionals enableNspawn [ systemd diff --git a/nixos/lib/test-driver/src/test_driver/__init__.py b/nixos/lib/test-driver/src/test_driver/__init__.py index 0f42f2842c77..fca8a135d082 100644 --- a/nixos/lib/test-driver/src/test_driver/__init__.py +++ b/nixos/lib/test-driver/src/test_driver/__init__.py @@ -147,9 +147,9 @@ def main() -> None: type=Path, ) arg_parser.add_argument( - "--dump-vsocks", + "--enable-ssh-backdoor", help="indicates that the interactive SSH backdoor is active and dumps information about it on start", - type=int, + action="store_true", ) args = arg_parser.parse_args() @@ -197,9 +197,10 @@ def main() -> None: keep_machine_state=args.keep_machine_state, global_timeout=args.global_timeout, debug=debugger, + enable_ssh_backdoor=args.enable_ssh_backdoor, ) as driver: - if offset := args.dump_vsocks: - driver.dump_machine_ssh(offset) + if args.enable_ssh_backdoor: + driver.dump_machine_ssh() if args.interactive: history_dir = os.getcwd() history_path = os.path.join(history_dir, ".nixos-test-history") diff --git a/nixos/lib/test-driver/src/test_driver/driver.py b/nixos/lib/test-driver/src/test_driver/driver.py index 4d1bb899e712..3233f6afc97e 100644 --- a/nixos/lib/test-driver/src/test_driver/driver.py +++ b/nixos/lib/test-driver/src/test_driver/driver.py @@ -8,6 +8,7 @@ import threading import traceback from collections.abc import Callable, Iterator from contextlib import AbstractContextManager, contextmanager +from dataclasses import dataclass from pathlib import Path from typing import Any from unittest import TestCase @@ -63,6 +64,45 @@ def pythonize_name(name: str) -> str: return re.sub(r"^[^A-Za-z_]|[^A-Za-z0-9_]", "_", name) +@dataclass +class VsockPair: + guest: Path + host: Path + cid: int + + +class VHostDeviceVsock: + def __init__(self, tmp_dir: Path, machines: list[str]): + self.temp_dir_handle = tempfile.TemporaryDirectory(dir=tmp_dir) + self.temp_dir = Path(self.temp_dir_handle.name) + self.sockets = { + machine: VsockPair( + self.temp_dir / f"{machine}_guest.socket", + self.temp_dir / f"{machine}_host.socket", + cid, + ) + for cid, machine in enumerate(machines, start=3) + } + + self.vhost_proc = subprocess.Popen( + [ + "vhost-device-vsock", + *( + arg + for vsock_pair in self.sockets.values() + for arg in ( + "--vm", + f"guest-cid={vsock_pair.cid},socket={vsock_pair.guest},uds-path={vsock_pair.host}", + ) + ), + ] + ) + + def __del__(self) -> None: + self.vhost_proc.kill() + self.temp_dir_handle.cleanup() + + class Driver: """A handle to the driver that sets up the environment and runs the tests""" @@ -80,6 +120,8 @@ class Driver: keep_machine_state: bool logger: AbstractLogger debug: DebugAbstract + vhost_vsock: VHostDeviceVsock | None = None + enable_ssh_backdoor: bool def __init__( self, @@ -94,6 +136,7 @@ class Driver: keep_machine_state: bool = False, global_timeout: int = 24 * 60 * 60 * 7, debug: DebugAbstract = DebugNop(), + enable_ssh_backdoor: bool = False, ): self.tests = tests self.out_dir = out_dir @@ -108,6 +151,7 @@ class Driver: self.container_start_scripts = dict( zip(container_names, container_start_scripts) ) + self.enable_ssh_backdoor = enable_ssh_backdoor def __enter__(self) -> "Driver": self.race_timer = threading.Timer(self.global_timeout, self.terminate_test) @@ -118,6 +162,12 @@ class Driver: self.polling_conditions = [] + if self.enable_ssh_backdoor and self.vm_start_scripts: + with self.logger.nested("start vhost-device-vsock"): + self.vhost_vsock = VHostDeviceVsock( + tmp_dir, list(self.vm_start_scripts.keys()) + ) + self.machines_qemu = [ QemuMachine( name=name, @@ -127,6 +177,16 @@ class Driver: callbacks=[self.check_polling_conditions], out_dir=self.out_dir, logger=self.logger, + vsock_host=( + self.vhost_vsock.sockets[name].host + if self.vhost_vsock is not None + else None + ), + vsock_guest=( + self.vhost_vsock.sockets[name].guest + if self.vhost_vsock is not None + else None + ), ) for name, vm_start_script in self.vm_start_scripts.items() ] @@ -219,6 +279,14 @@ class Driver: except Exception as e: self.logger.error(f"Error during cleanup of vlan{vlan.nr}: {e}") + if self.enable_ssh_backdoor: + try: + del self.vhost_vsock + except Exception as e: + self.logger.error( + f"Error during cleanup of vhost-device-vsock process: {e}" + ) + def subtest(self, name: str) -> Iterator[None]: """Group logs under a given test name""" with self.logger.subtest(name): @@ -256,6 +324,7 @@ class Driver: NspawnMachine=NspawnMachine, # for typing t=AssertionTester(), debug=self.debug, + dump_machine_ssh=self.dump_machine_ssh, ) machine_symbols = {pythonize_name(m.name): m for m in self.machines} # If there's exactly one machine, make it available under the name @@ -275,18 +344,28 @@ class Driver: ) return {**general_symbols, **machine_symbols, **vlan_symbols} - def dump_machine_ssh(self, offset: int) -> None: - print("SSH backdoor enabled, the machines can be accessed like this:") - print( - f"{Style.BRIGHT}Note:{Style.RESET_ALL} vsocks require {Style.BRIGHT}systemd-ssh-proxy(1){Style.RESET_ALL} to be enabled (default on NixOS 25.05 and newer)." - ) - longest_name = len(max((machine.name for machine in self.machines), key=len)) - for index, machine in enumerate(self.machines, start=offset + 1): - name = machine.name - spaces = " " * (longest_name - len(name) + 2) + def dump_machine_ssh(self) -> None: + if not self.enable_ssh_backdoor: + return + + assert self.vhost_vsock is not None + + if self.machines: + print("SSH backdoor enabled, the machines can be accessed like this:") print( - f" {name}:{spaces}{Style.BRIGHT}{machine.ssh_backdoor_command(index)}{Style.RESET_ALL}" + f"{Style.BRIGHT}Note:{Style.RESET_ALL} this requires {Style.BRIGHT}systemd-ssh-proxy(1){Style.RESET_ALL} to be enabled (default on NixOS 25.05 and newer)." ) + longest_name = len( + max((machine.name for machine in self.machines), key=len) + ) + for index, machine in enumerate(self.machines): + name = machine.name + spaces = " " * (longest_name - len(name) + 2) + print( + f" {name}:{spaces}{Style.BRIGHT}{machine.ssh_backdoor_command()}{Style.RESET_ALL}" + ) + else: + print("SSH backdoor enabled, but no machines defined") def test_script(self) -> None: """Run the test script""" @@ -386,6 +465,10 @@ class Driver: """ tmp_dir = get_tmp_dir() + if self.enable_ssh_backdoor: + self.logger.warning( + f"create_machine({name}): not enabling SSH backdoor, this is not supported for VMs created with create_machine!" + ) return QemuMachine( tmp_dir=tmp_dir, out_dir=self.out_dir, diff --git a/nixos/lib/test-driver/src/test_driver/machine/__init__.py b/nixos/lib/test-driver/src/test_driver/machine/__init__.py index 365826dc22ff..c384d5cd0457 100644 --- a/nixos/lib/test-driver/src/test_driver/machine/__init__.py +++ b/nixos/lib/test-driver/src/test_driver/machine/__init__.py @@ -147,6 +147,7 @@ class QemuStartCommand: qmp_socket_path: Path, shell_socket_path: Path, allow_reboot: bool = False, + vsock_guest: Path | None = None, ) -> str: display_opts = "" @@ -170,6 +171,12 @@ class QemuStartCommand: if not allow_reboot: qemu_opts += " -no-reboot" + if vsock_guest is not None: + qemu_opts += ( + f" -chardev socket,id=vsock_ssh,path={vsock_guest} " + f"-device vhost-user-vsock-pci,chardev=vsock_ssh " + ) + return ( f"{self._cmd}" f" -qmp unix:{qmp_socket_path},server=on,wait=off" @@ -203,10 +210,15 @@ class QemuStartCommand: qmp_socket_path: Path, shell_socket_path: Path, allow_reboot: bool, + vsock_guest: Path | None = None, ) -> subprocess.Popen: return subprocess.Popen( self.cmd( - monitor_socket_path, qmp_socket_path, shell_socket_path, allow_reboot + monitor_socket_path, + qmp_socket_path, + shell_socket_path, + allow_reboot, + vsock_guest, ), stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -724,6 +736,9 @@ class QemuMachine(BaseMachine): shell: socket.socket | None serial_thread: threading.Thread | None + vsock_guest: Path | None + vsock_host: Path | None + booted: bool connected: bool # Store last serial console lines for use @@ -741,6 +756,8 @@ class QemuMachine(BaseMachine): name: str | None = None, keep_machine_state: bool = False, callbacks: list[Callable] | None = None, + vsock_guest: Path | None = None, + vsock_host: Path | None = None, ) -> None: self.start_command = QemuStartCommand(start_command) super().__init__( @@ -753,6 +770,8 @@ class QemuMachine(BaseMachine): ) self.full_console_log = [] + self.vsock_guest = vsock_guest + self.vsock_host = vsock_host # set up directories self.monitor_path = self.state_dir / "monitor" @@ -769,8 +788,9 @@ class QemuMachine(BaseMachine): self.booted = False self.connected = False - def ssh_backdoor_command(self, index: int) -> str: - return f"ssh -o User=root vsock/{index}" + def ssh_backdoor_command(self) -> str: + assert self.vsock_host is not None + return f"ssh -o User=root vsock-mux/{self.vsock_host}" def is_up(self) -> bool: return self.booted and self.connected @@ -1224,6 +1244,7 @@ class QemuMachine(BaseMachine): self.qmp_path, self.shell_path, allow_reboot, + self.vsock_guest, ) self.monitor, _ = monitor_socket.accept() self.shell, _ = shell_socket.accept() @@ -1434,7 +1455,7 @@ class NspawnMachine(BaseMachine): self.machine_sock_path = self.tmp_dir / f"{self.name}-nspawn.sock" - def ssh_backdoor_command(self, index: int) -> str: + def ssh_backdoor_command(self) -> str: # documented in systemd-ssh-generator(8) and https://systemd.io/CONTAINER_INTERFACE/ socket_path = f"/run/systemd/nspawn/unix-export/{self.name}/ssh" proxy_cmd = f"socat - UNIX-CLIENT:{socket_path}" diff --git a/nixos/lib/test-script-prepend.py b/nixos/lib/test-script-prepend.py index e836e779300c..b1ca8e7b1678 100644 --- a/nixos/lib/test-script-prepend.py +++ b/nixos/lib/test-script-prepend.py @@ -58,3 +58,4 @@ serial_stdout_on: Callable[[], None] polling_condition: PollingConditionProtocol debug: DebugAbstract t: TestCase +dump_machine_ssh: Callable[[], None] diff --git a/nixos/lib/testing/driver.nix b/nixos/lib/testing/driver.nix index 5cd7c2ebb806..fa3abf5c0c7a 100644 --- a/nixos/lib/testing/driver.nix +++ b/nixos/lib/testing/driver.nix @@ -7,6 +7,8 @@ let inherit (lib) mkOption types literalMD; + inherit (config) sshBackdoor; + # Reifies and correctly wraps the python test driver for # the respective qemu version and with or without ocr support testDriver = config.pythonTestDriverPackage.override { @@ -232,5 +234,27 @@ in # make available on the test runner passthru.driver = config.driver; + + nodeDefaults = + { config, ... }: + { + # This is needed for the SSH backdoor to function. + # Set this to `true` by default to not change essential QEMU flags + # depending on whether debugging is enabled. + # + # If needed, this can still be turned off. + virtualisation.qemu.enableSharedMemory = lib.mkDefault true; + + assertions = [ + { + assertion = sshBackdoor.enable -> config.virtualisation.qemu.enableSharedMemory; + message = '' + When turning on the SSH backdoor of the NixOS test-framework, + `virtualisation.qemu.enableSharedMemory` MUST be `true` + (affected: ${config.networking.hostName}). + ''; + } + ]; + }; }; } diff --git a/nixos/lib/testing/nodes.nix b/nixos/lib/testing/nodes.nix index 7384dec68895..ab20d8505354 100644 --- a/nixos/lib/testing/nodes.nix +++ b/nixos/lib/testing/nodes.nix @@ -13,6 +13,7 @@ let mapAttrs mkIf mkMerge + mkRemovedOptionModule mkOption optionalAttrs types @@ -128,6 +129,12 @@ let in { + imports = [ + (mkRemovedOptionModule [ "sshBackdoor" "vsockOffset" ] '' + The option `sshBackdoor.vsockOffset` has been removed from the testing framework. + The functionality provided by it is not needed anymore. + '') + ]; options = { sshBackdoor = { @@ -137,22 +144,6 @@ in type = types.bool; description = "Whether to turn on the VSOCK-based access to all VMs. This provides an unauthenticated access intended for debugging."; }; - vsockOffset = mkOption { - default = 2; - type = types.ints.between 2 4294967296; - description = '' - This field is only relevant when multiple users run the (interactive) - driver outside the sandbox and with the SSH backdoor activated. - The typical symptom for this being a problem are error messages like this: - `vhost-vsock: unable to set guest cid: Address already in use` - - This option allows to assign an offset to each vsock number to - resolve this. - - This is a 32bit number. The lowest possible vsock number is `3` - (i.e. with the lowest node number being `1`, this is 2+1). - ''; - }; }; node.type = mkOption { @@ -318,7 +309,7 @@ in passthru.containers = config.containers; extraDriverArgs = mkIf config.sshBackdoor.enable [ - "--dump-vsocks=${toString config.sshBackdoor.vsockOffset}" + "--enable-ssh-backdoor" ]; defaults = mkMerge [ @@ -341,20 +332,6 @@ in }) ]; - nodeDefaults = mkIf config.sshBackdoor.enable ( - let - inherit (config.sshBackdoor) vsockOffset; - in - { config, ... }: - { - virtualisation.qemu.options = [ - "-device vhost-vsock-pci,guest-cid=${ - toString (config.virtualisation.test.nodeNumber + vsockOffset) - }" - ]; - } - ); - # Docs: nixos/doc/manual/development/writing-nixos-tests.section.md /** See https://nixos.org/manual/nixos/unstable#sec-override-nixos-test diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 9a059c30563c..6a3f2875a318 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -149,6 +149,7 @@ in lib-extend = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./nixos-test-driver/lib-extend.nix { }; node-name = runTest ./nixos-test-driver/node-name.nix; busybox = runTest ./nixos-test-driver/busybox.nix; + ssh-backdoor = runTestOn [ "x86_64-linux" ] ./nixos-test-driver/ssh-backdoor.nix; console-log = runTest ./nixos-test-driver/console-log.nix; containers = runTest ./nixos-test-driver/containers.nix; driver-timeout = diff --git a/nixos/tests/nixos-test-driver/ssh-backdoor.nix b/nixos/tests/nixos-test-driver/ssh-backdoor.nix new file mode 100644 index 000000000000..5c84b24474e9 --- /dev/null +++ b/nixos/tests/nixos-test-driver/ssh-backdoor.nix @@ -0,0 +1,36 @@ +{ pkgs, lib, ... }: +{ + name = "ssh-backdoor"; + sshBackdoor.enable = true; + + nodes.machine = { }; + + testScript = '' + import subprocess + + start_all() + machine.wait_for_unit("multi-user.target") + + assert driver.vhost_vsock is not None + host_socket = driver.vhost_vsock.sockets["machine"].host + + with subtest("ssh from the host via systemd-ssh-proxy"): + subprocess.run( + [ + "${lib.getExe pkgs.openssh}", + "-vvv", + f"vsock-mux/{host_socket}", + "-o", + "User=root", + "-F", + # The backdoor feature of the driver copies this into NIX_BUILD_TOP. + # We can't do this here since `enableDebugHook=true;` would halt + # instead of terminating the test execution if it fails. + "${pkgs.systemd}/lib/systemd/ssh_config.d/20-systemd-ssh-proxy.conf", + "--", + "true" + ], + check=True + ) + ''; +}