nixos/tests/incus: present instances as a submodule

nixos/modules/virtualisation/incus-virtual-machine.nix

@@ -25,7 +25,7 @@ in
 
       partitionTableType = "efi";
       format = "qcow2-compressed";
-      copyChannel = true;
+      copyChannel = config.system.installer.channel.enable;
     };
 
     fileSystems = {

nixos/modules/virtualisation/lxc-instance-common.nix

@@ -27,7 +27,7 @@
   services.openssh.startWhenNeeded = lib.mkDefault true;
 
   # As this is intended as a standalone image, undo some of the minimal profile stuff
-  documentation.enable = true;
-  documentation.nixos.enable = true;
+  documentation.enable = lib.mkDefault true;
+  documentation.nixos.enable = lib.mkDefault true;
   services.logrotate.enable = true;
 }

nixos/tests/all-tests.nix

@@ -744,11 +744,11 @@ in
   incron = runTest ./incron.nix;
   incus = import ./incus {
     inherit runTest;
-    lts = false;
+    package = pkgs.incus;
   };
   incus-lts = import ./incus {
     inherit runTest;
-    lts = true;
+    package = pkgs.incus-lts;
   };
   influxdb = runTest ./influxdb.nix;
   influxdb2 = runTest ./influxdb2.nix;

nixos/tests/incus/default.nix

@@ -1,6 +1,6 @@
 {
+  package,
   runTest,
-  lts ? true,
 }:
 let
   incusRunTest =
@@ -12,32 +12,40 @@ let
       ];
 
       tests.incus = {
-        inherit lts;
+        inherit package;
       }
       // config;
     };
 in
 {
-  all = incusRunTest { all = true; };
+  # appArmor = incusRunTest {
+  #   all = true;
+  #   appArmor = true;
+  # };
 
-  appArmor = incusRunTest {
-    all = true;
-    appArmor = true;
+  container = incusRunTest {
+    instances.c1 = {
+      type = "container";
+    };
   };
 
-  container = incusRunTest { instance.container = true; };
-
-  lvm = incusRunTest { storage.lvm = true; };
-
-  openvswitch = incusRunTest { network.ovs = true; };
+  # lvm = incusRunTest { storage.lvm = true; };
+  #
+  # openvswitch = incusRunTest { network.ovs = true; };
 
   ui = runTest {
     imports = [ ./ui.nix ];
 
-    _module.args = { inherit lts; };
+    _module.args = { inherit package; };
   };
 
-  virtual-machine = incusRunTest { instance.virtual-machine = true; };
+  virtual-machine = incusRunTest {
+    instances = {
+      vm1 = {
+        type = "virtual-machine";
+      };
+    };
+  };
 
   zfs = incusRunTest { storage.zfs = true; };
 }

nixos/tests/incus/incus-tests-module.nix

@@ -1,26 +1,248 @@
-{ config, lib, ... }:
+{
+  lib,
+  pkgs,
+  ...
+}:
 let
-  cfg = config.tests.incus;
+  jsonFormat = pkgs.formats.json { };
 in
 {
   options.tests.incus = {
-    lts = lib.mkEnableOption "LTS package testing";
+    package = lib.mkPackageOption pkgs "incus" { };
+
+    preseed = lib.mkOption {
+      description = "configuration provided to incus preseed. https://linuxcontainers.org/incus/docs/main/howto/initialize/#non-interactive-configuration";
+      type = lib.types.submodule {
+        freeformType = jsonFormat.type;
+      };
+    };
+
+    instances = lib.mkOption {
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, config, ... }:
+          {
+            options = {
+              name = lib.mkOption {
+                type = lib.types.str;
+                default = name;
+              };
+
+              type = lib.mkOption {
+                type = lib.types.enum [
+                  "container"
+                  "virtual-machine"
+                ];
+
+              };
+
+              imageAlias = lib.mkOption {
+                type = lib.types.str;
+                description = "name of image when imported";
+                default = "nixos/${name}/${config.type}";
+              };
+
+              nixosConfig = lib.mkOption {
+                type = lib.types.attrsOf lib.types.anything;
+                default = { };
+              };
+
+              incusConfig = lib.mkOption {
+                type = lib.types.submodule {
+                  freeformType = jsonFormat.type;
+                };
+                description = "incus configuration provided at launch";
+                default = { };
+              };
+
+              testScript = lib.mkOption {
+                type = lib.types.str;
+                description = "final script provided to test runner";
+                readOnly = true;
+              };
+            };
+            config =
+              let
+                releases = import ../../release.nix {
+                  configuration = config.nixosConfig;
+                };
+
+                images = {
+                  container = {
+                    metadata =
+                      releases.incusContainerMeta.${pkgs.stdenv.hostPlatform.system}
+                      + "/tarball/nixos-image-lxc-*-${pkgs.stdenv.hostPlatform.system}.tar.xz";
+
+                    root =
+                      releases.incusContainerImage.${pkgs.stdenv.hostPlatform.system}
+                      + "/nixos-lxc-image-${pkgs.stdenv.hostPlatform.system}.squashfs";
+                  };
+
+                  virtual-machine = {
+                    metadata = releases.incusVirtualMachineImageMeta.${pkgs.stdenv.hostPlatform.system} + "/*/*.tar.xz";
+                    root = releases.incusVirtualMachineImage.${pkgs.stdenv.hostPlatform.system} + "/nixos.qcow2";
+                  };
+                };
+
+                root = images.${config.type}.root;
+                metadata = images.${config.type}.metadata;
+
+                image_id = "${config.type}/${config.name}";
+              in
+              {
+                incusConfig = lib.optionalAttrs (config.type == "virtual-machine") {
+                  config."security.secureboot" = false;
+                };
+
+                nixosConfig = {
+                  # Building documentation makes the test unnecessarily take a longer time:
+                  documentation.enable = lib.mkForce false;
+                  documentation.nixos.enable = lib.mkForce false;
+                  # including a channel forces images to be rebuilt on any changes
+                  system.installer.channel.enable = lib.mkForce false;
+
+                  environment.etc."nix/registry.json".text = lib.mkForce "{}";
+
+                  # Arbitrary sysctl setting changed from nixos default
+                  # used for verifying `distrobuilder.generator` properly allows
+                  # for containers to modify sysctl
+                  boot.kernel.sysctl."net.ipv4.ip_forward" = "1";
+                };
+
+                testScript = # python
+                ''
+                  with subtest("[${image_id}] image can be imported"):
+                      server.succeed("incus image import ${metadata} ${root} --alias ${config.imageAlias}")
+
+                  with subtest("[${image_id}] can be launched and managed"):
+                      instance_name = server.succeed("incus launch ${config.imageAlias}${
+                        lib.optionalString (config.type == "virtual-machine") " --vm"
+                      } --quiet < ${jsonFormat.generate "${config.name}.json" config.incusConfig}").split(":")[1].strip()
+                      server.wait_for_instance(instance_name)
+
+                  with subtest("[${image_id}] can successfully restart"):
+                      server.succeed(f"incus restart {instance_name}")
+                      server.wait_for_instance(instance_name)
+
+                  with subtest("[${image_id}] remains running when softDaemonRestart is enabled and service is stopped"):
+                      pid = server.succeed(f"incus info {instance_name} | grep 'PID'").split(":")[1].strip()
+                      server.succeed(f"ps {pid}")
+                      server.succeed("systemctl stop incus")
+                      server.succeed(f"ps {pid}")
+                      server.succeed("systemctl start incus")
+
+                  with subtest("[${image_id}] CPU limits can be managed"):
+                      server.set_instance_config(instance_name, "limits.cpu 1", restart=True)
+                      server.wait_instance_exec_success(instance_name, "nproc | grep '^1$'", timeout=90)
+
+                  with subtest("[${image_id}] CPU limits can be hotplug changed"):
+                      server.set_instance_config(instance_name, "limits.cpu 2")
+                      server.wait_instance_exec_success(instance_name, "nproc | grep '^2$'", timeout=90)
+
+                  with subtest("[${image_id}] exec has a valid path"):
+                      server.succeed(f"incus exec {instance_name} -- bash -c 'true'")
+
+                  with subtest("[${image_id}] software tpm can be configured"):
+                      # this can be hot added to containers, but stopping for vm
+                      server.succeed(f"incus stop {instance_name}")
+                      server.succeed(f"incus config device add {instance_name} vtpm tpm path=/dev/tpm0 pathrm=/dev/tpmrm0")
+                      server.succeed(f"incus start {instance_name}")
+                      server.wait_for_instance(instance_name)
+
+                      server.succeed(f"incus exec {instance_name} -- test -e /dev/tpm0")
+                      server.succeed(f"incus exec {instance_name} -- test -e /dev/tpmrm0")
+                ''
+                #
+                # container specific
+                #
+                +
+                  lib.optionalString (config.type == "container")
+                    # python
+                    ''
+                      # TODO troubleshoot VM hot memory resizing which was introduced in 6.12
+                      with subtest("[${image_id}] memory limits can be hotplug changed"):
+                          server.set_instance_config(instance_name, "limits.memory 512MB")
+                          # can't use lsmem since it sees the host's memory size
+                          server.wait_instance_exec_success(instance_name, "grep 'MemTotal:[[:space:]]*500000 kB' /proc/meminfo", timeout=1)
+
+                      # verify the patched container systemd generator from `pkgs.distrobuilder.generator`
+                      with subtest("[${image_id}] lxc-generator compatibility"):
+                          with subtest("[${image_id}] lxc-container generator configures plain container"):
+                              # default container is plain
+                              server.succeed(f"incus exec {instance_name} test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
+
+                              server.check_instance_sysctl(instance_name)
+
+                          with subtest("[${image_id}] lxc-container generator configures nested container"):
+                              server.set_instance_config(instance_name, "security.nesting=true", restart=True)
+
+                              server.fail(f"incus exec {instance_name} test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
+                              target = server.succeed(f"incus exec {instance_name} readlink -- -f /run/systemd/system/systemd-binfmt.service").strip()
+                              assert target == "/dev/null", "lxc generator did not correctly mask /run/systemd/system/systemd-binfmt.service"
+
+                              server.check_instance_sysctl(instance_name)
+
+                      with subtest("[${image_id}] lxcfs"):
+                          with subtest("[${image_id}] mounts lxcfs overlays"):
+                              server.succeed(f"incus exec {instance_name} mount | grep 'lxcfs on /proc/cpuinfo type fuse.lxcfs'")
+                              server.succeed(f"incus exec {instance_name} mount | grep 'lxcfs on /proc/meminfo type fuse.lxcfs'")
+
+                          with subtest("[${image_id}] supports per-instance lxcfs"):
+                              server.succeed(f"incus stop {instance_name}")
+                              server.fail(f"pgrep -a lxcfs | grep 'incus/devices/{instance_name}/lxcfs'")
+
+                              server.succeed("incus config set instances.lxcfs.per_instance=true")
+
+                              server.succeed(f"incus start {instance_name}")
+                              server.wait_for_instance(instance_name)
+                              server.succeed(f"pgrep -a lxcfs | grep 'incus/devices/{instance_name}/lxcfs'")
+                    ''
+
+                #
+                # virtual-machine specific
+                #
+                +
+                  lib.optionalString (config.type == "virtual-machine")
+                    # python
+                    ''
+                      with subtest("[${image_id}] memory limits can be managed"):
+                          server.set_instance_config(instance_name, "limits.memory 384MB", restart=True)
+                          lsmem = json.loads(server.instance_succeed(instance_name, "lsmem --json"))
+                          memsize = lsmem["memory"][0]["size"]
+                          assert memsize == "384M", f"failed to manage memory limit. {memsize} != 384M"
+
+                      with subtest("[${image_id}] incus-agent is started"):
+                          server.succeed(f"incus exec {instance_name} systemctl is-active incus-agent")
+                    ''
+
+                +
+                  #
+                  # finalize
+                  #
+                  # python
+                  ''
+                    # this will leave the instances stopped
+                    with subtest("[${image_id}] stop with incus-startup.service"):
+                        pid = server.succeed(f"incus info {instance_name} | grep 'PID'").split(":")[1].strip()
+                        server.succeed(f"ps {pid}")
+                        server.succeed("systemctl stop incus-startup.service")
+                        server.wait_until_fails(f"ps {pid}", timeout=120)
+                        server.succeed("systemctl start incus-startup.service")
+
+                  '';
+
+              };
+          }
+        )
+      );
+      description = "";
+      default = { };
+    };
 
-    all = lib.mkEnableOption "All tests";
     appArmor = lib.mkEnableOption "AppArmor during tests";
 
     feature.user = lib.mkEnableOption "Validate incus user access feature";
 
-    init = {
-      legacy = lib.mkEnableOption "Validate non-systemd init";
-      systemd = lib.mkEnableOption "Validate systemd init";
-    };
-
-    instance = {
-      container = lib.mkEnableOption "Validate container functionality";
-      virtual-machine = lib.mkEnableOption "Validate virtual machine functionality";
-    };
-
     network.ovs = lib.mkEnableOption "Validate OVS network integration";
 
     storage = {
@@ -30,27 +252,6 @@ in
   };
 
   config = {
-    tests.incus = {
-      lts = lib.mkDefault true;
-
-      feature.user = lib.mkDefault cfg.all;
-
-      init = {
-        legacy = lib.mkDefault cfg.all;
-        systemd = lib.mkDefault true;
-      };
-
-      instance = {
-        container = lib.mkDefault cfg.all;
-        virtual-machine = lib.mkDefault cfg.all;
-      };
-
-      network.ovs = lib.mkDefault cfg.all;
-
-      storage = {
-        lvm = lib.mkDefault cfg.all;
-        zfs = lib.mkDefault cfg.all;
-      };
-    };
+    tests.incus = { };
   };
 }

nixos/tests/incus/incus-tests.nix

@@ -5,58 +5,33 @@
   ...
 }:
 
+# TODO aarch64 vm filter
+# TODO CSM
+# with subtest("incus-user allows users to create instances"):
+#     server.succeed("su - testuser bash -c 'incus image import ${(images "systemd").container.metadata} ${(images "systemd").container.rootfs} --alias nixos'")
+#     server.succeed("su - testuser bash -c 'incus launch nixos instance2'")
+#     server.wait_for_instance("instance2", "user-1000")
+#
 let
   cfg = config.tests.incus;
 
-  releases =
-    init:
-    import ../../release.nix {
-      configuration = {
-        # Building documentation makes the test unnecessarily take a longer time:
-        documentation.enable = lib.mkForce false;
-
-        boot.initrd.systemd.enable = init == "systemd";
-
-        # Arbitrary sysctl modification to ensure containers can update sysctl
-        boot.kernel.sysctl."net.ipv4.ip_forward" = "1";
-      };
-    };
-
-  images = init: {
-    container = {
-      metadata =
-        (releases init).incusContainerMeta.${pkgs.stdenv.hostPlatform.system}
-        + "/tarball/nixos-image-lxc-*-${pkgs.stdenv.hostPlatform.system}.tar.xz";
-
-      rootfs =
-        (releases init).incusContainerImage.${pkgs.stdenv.hostPlatform.system}
-        + "/nixos-lxc-image-${pkgs.stdenv.hostPlatform.system}.squashfs";
-    };
-
-    virtual-machine = {
-      metadata =
-        (releases init).incusVirtualMachineImageMeta.${pkgs.stdenv.hostPlatform.system} + "/*/*.tar.xz";
-      disk = (releases init).incusVirtualMachineImage.${pkgs.stdenv.hostPlatform.system} + "/nixos.qcow2";
-    };
-  };
-
-  initVariants =
-    lib.optionals cfg.init.legacy [ "legacy" ] ++ lib.optionals cfg.init.systemd [ "systemd" ];
-
-  canTestVm = cfg.instance.virtual-machine && pkgs.stdenv.isLinux && pkgs.stdenv.isx86_64;
+  instanceScript = lib.foldlAttrs (
+    acc: name: instance:
+    acc + instance.testScript
+  ) "" cfg.instances;
 in
 {
-  name = "incus" + lib.optionalString cfg.lts "-lts";
+  name = cfg.package.name;
 
   meta = {
     maintainers = lib.teams.lxc.members;
   };
 
-  nodes.machine = {
+  nodes.server = {
     virtualisation = {
       cores = 2;
       memorySize = 4096;
-      diskSize = 12 * 1024;
+      diskSize = 20 * 1024;
       emptyDiskImages = [
         # vdb for zfs
         2048
@@ -66,7 +41,7 @@ in
 
       incus = {
         enable = true;
-        package = if cfg.lts then pkgs.incus-lts else pkgs.incus;
+        package = cfg.package;
 
         preseed = {
           networks = [
@@ -152,280 +127,83 @@ in
       ''
         server = IncusHost(machine)
       ''
-    + lib.optionalString cfg.appArmor ''
-      with subtest("Verify AppArmor service is started without issue"):
-          # restart AppArmor service since the Incus AppArmor folders are
-          # created after AA service is started
-          machine.systemctl("restart apparmor.service")
-          machine.succeed("systemctl --no-pager -l status apparmor.service")
-          machine.wait_for_unit("apparmor.service")
-    ''
-    + lib.optionalString cfg.instance.container (
-      lib.foldl (
-        acc: variant:
-        acc
-        # python
-        + ''
-          metadata = "${(images variant).container.metadata}"
-          rootfs = "${(images variant).container.rootfs}"
-          alias = "nixos/container/${variant}"
-          variant = "${variant}"
-
-          with subtest("container image can be imported"):
-              machine.succeed(f"incus image import {metadata} {rootfs} --alias {alias}")
-
-
-          with subtest("container can be launched and managed"):
-              machine.succeed(f"incus launch {alias} container-{variant}1")
-              machine.wait_for_instance(f"container-{variant}1")
-
-
-          with subtest("container mounts lxcfs overlays"):
-              machine.succeed(f"incus exec container-{variant}1 mount | grep 'lxcfs on /proc/cpuinfo type fuse.lxcfs'")
-              machine.succeed(f"incus exec container-{variant}1 mount | grep 'lxcfs on /proc/meminfo type fuse.lxcfs'")
-
-
-          with subtest("container CPU limits can be managed"):
-              machine.set_instance_config(f"container-{variant}1", "limits.cpu 1", restart=True)
-              machine.wait_instance_exec_success(f"container-{variant}1", "nproc | grep '^1$'", timeout=90)
-
-
-          with subtest("container CPU limits can be hotplug changed"):
-              machine.set_instance_config(f"container-{variant}1", "limits.cpu 2")
-              machine.wait_instance_exec_success(f"container-{variant}1", "nproc | grep '^2$'", timeout=90)
-
-
-          with subtest("container memory limits can be managed"):
-              machine.set_instance_config(f"container-{variant}1", "limits.memory 128MB", restart=True)
-              machine.wait_instance_exec_success(f"container-{variant}1", "grep 'MemTotal:[[:space:]]*125000 kB' /proc/meminfo", timeout=90)
-
-
-          with subtest("container memory limits can be hotplug changed"):
-              machine.set_instance_config(f"container-{variant}1", "limits.memory 256MB")
-              machine.wait_instance_exec_success(f"container-{variant}1", "grep 'MemTotal:[[:space:]]*250000 kB' /proc/meminfo", timeout=90)
-
-
-          with subtest("container software tpm can be configured"):
-              machine.succeed(f"incus config device add container-{variant}1 vtpm tpm path=/dev/tpm0 pathrm=/dev/tpmrm0")
-              machine.succeed(f"incus exec container-{variant}1 -- test -e /dev/tpm0")
-              machine.succeed(f"incus exec container-{variant}1 -- test -e /dev/tpmrm0")
-              machine.succeed(f"incus config device remove container-{variant}1 vtpm")
-              machine.fail(f"incus exec container-{variant}1 -- test -e /dev/tpm0")
-
-
-          with subtest("container lxc-generator compatibility"):
-              with subtest("lxc-container generator configures plain container"):
-                  # default container is plain
-                  machine.succeed(f"incus exec container-{variant}1 test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
-
-                  machine.check_instance_sysctl(f"container-{variant}1")
-
-              with subtest("lxc-container generator configures nested container"):
-                  machine.set_instance_config(f"container-{variant}1", "security.nesting=true", restart=True)
-
-                  machine.fail(f"incus exec container-{variant}1 test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
-                  target = machine.succeed(f"incus exec container-{variant}1 readlink -- -f /run/systemd/system/systemd-binfmt.service").strip()
-                  assert target == "/dev/null", "lxc generator did not correctly mask /run/systemd/system/systemd-binfmt.service"
-
-                  machine.check_instance_sysctl(f"container-{variant}1")
-
-              with subtest("lxc-container generator configures privileged container"):
-                  # Create a new instance for a clean state
-                  machine.succeed(f"incus launch {alias} container-{variant}2")
-                  machine.wait_for_instance(f"container-{variant}2")
-
-                  machine.succeed(f"incus exec container-{variant}2 test -- -e /run/systemd/system/service.d/zzz-lxc-service.conf")
-
-                  machine.check_instance_sysctl(f"container-{variant}2")
-
-          with subtest("container supports per-instance lxcfs"):
-              machine.succeed(f"incus stop container-{variant}1")
-              machine.fail(f"pgrep -a lxcfs | grep 'incus/devices/container-{variant}1/lxcfs'")
-
-              machine.succeed("incus config set instances.lxcfs.per_instance=true")
-
-              machine.succeed(f"incus start container-{variant}1")
-              machine.wait_for_instance(f"container-{variant}1")
-              machine.succeed(f"pgrep -a lxcfs | grep 'incus/devices/container-{variant}1/lxcfs'")
-
-
-          with subtest("container can successfully restart"):
-              machine.succeed(f"incus restart container-{variant}1")
-              machine.wait_for_instance(f"container-{variant}1")
-
-
-          with subtest("container remains running when softDaemonRestart is enabled and service is stopped"):
-              pid = machine.succeed(f"incus info container-{variant}1 | grep 'PID'").split(":")[1].strip()
-              machine.succeed(f"ps {pid}")
-              machine.succeed("systemctl stop incus")
-              machine.succeed(f"ps {pid}")
-              machine.succeed("systemctl start incus")
-
-              with subtest("containers stop with incus-startup.service"):
-                  pid = machine.succeed(f"incus info container-{variant}1 | grep 'PID'").split(":")[1].strip()
-                  machine.succeed(f"ps {pid}")
-                  machine.succeed("systemctl stop incus-startup.service")
-                  machine.wait_until_fails(f"ps {pid}", timeout=120)
-                  machine.succeed("systemctl start incus-startup.service")
-
-
-          machine.cleanup()
-        ''
-      ) "" initVariants
-    )
-    + lib.optionalString canTestVm (
-      (lib.foldl (
-        acc: variant:
-        acc
-        # python
-        + ''
-          metadata = "${(images variant).virtual-machine.metadata}"
-          disk = "${(images variant).virtual-machine.disk}"
-          alias = "nixos/virtual-machine/${variant}"
-          variant = "${variant}"
-
-          with subtest("virtual-machine image can be imported"):
-              machine.succeed(f"incus image import {metadata} {disk} --alias {alias}")
-
-
-          with subtest("virtual-machine can be created"):
-              machine.succeed(f"incus create {alias} vm-{variant}1 --vm --config limits.memory=512MB --config security.secureboot=false")
-
-
-          with subtest("virtual-machine software tpm can be configured"):
-              machine.succeed(f"incus config device add vm-{variant}1 vtpm tpm path=/dev/tpm0")
-
-
-          with subtest("virtual-machine can be launched and become available"):
-              machine.succeed(f"incus start vm-{variant}1")
-              machine.wait_for_instance(f"vm-{variant}1")
-
-
-          with subtest("virtual-machine incus-agent is started"):
-              machine.succeed(f"incus exec vm-{variant}1 systemctl is-active incus-agent")
-
-
-          with subtest("virtual-machine incus-agent has a valid path"):
-              machine.succeed(f"incus exec vm-{variant}1 -- bash -c 'true'")
-
-
-          with subtest("virtual-machine CPU limits can be managed"):
-              machine.set_instance_config(f"vm-{variant}1", "limits.cpu 1", restart=True)
-              machine.wait_instance_exec_success(f"vm-{variant}1", "nproc | grep '^1$'", timeout=90)
-
-
-          with subtest("virtual-machine CPU limits can be hotplug changed"):
-              machine.set_instance_config(f"vm-{variant}1", "limits.cpu 2")
-              machine.wait_instance_exec_success(f"vm-{variant}1", "nproc | grep '^2$'", timeout=90)
-
-
-          with subtest("virtual-machine can successfully restart"):
-              machine.succeed(f"incus restart vm-{variant}1")
-              machine.wait_for_instance(f"vm-{variant}1")
-
-
-          with subtest("virtual-machine remains running when softDaemonRestart is enabled and service is stopped"):
-              pid = machine.succeed(f"incus info vm-{variant}1 | grep 'PID'").split(":")[1].strip()
-              machine.succeed(f"ps {pid}")
-              machine.succeed("systemctl stop incus")
-              machine.succeed(f"ps {pid}")
-              machine.succeed("systemctl start incus")
-
-
-              with subtest("virtual-machines stop with incus-startup.service"):
-                  pid = machine.succeed(f"incus info vm-{variant}1 | grep 'PID'").split(":")[1].strip()
-                  machine.succeed(f"ps {pid}")
-                  machine.succeed("systemctl stop incus-startup.service")
-                  machine.wait_until_fails(f"ps {pid}", timeout=120)
-                  machine.succeed("systemctl start incus-startup.service")
-
-
-          machine.cleanup()
-        ''
-      ) "" initVariants)
-      +
-        # python
-        ''
-          with subtest("virtual-machine can launch CSM (BIOS)"):
-              machine.succeed("incus init csm --vm --empty -c security.csm=true -c security.secureboot=false")
-              machine.succeed("incus start csm")
-
-
-          machine.cleanup()
-        ''
-    )
-    +
-      lib.optionalString cfg.feature.user # python
-        ''
-          with subtest("incus-user allows restricted access for users"):
-              machine.fail("incus project show user-1000")
-              machine.succeed("su - testuser bash -c 'incus list'")
-              # a project is created dynamically for the user
-              machine.succeed("incus project show user-1000")
-              # users shouldn't be able to list storage pools
-              machine.fail("su - testuser bash -c 'incus storage list'")
-
-
-          with subtest("incus-user allows users to launch instances"):
-              machine.succeed("su - testuser bash -c 'incus image import ${(images "systemd").container.metadata} ${(images "systemd").container.rootfs} --alias nixos'")
-              machine.succeed("su - testuser bash -c 'incus launch nixos instance2'")
-              machine.wait_for_instance("instance2", "user-1000")
-
-          machine.cleanup()
-        ''
     +
       lib.optionalString cfg.network.ovs # python
         ''
           with subtest("Verify openvswitch bridge"):
-              machine.succeed("incus network info ovsbr0")
+              server.succeed("incus network info ovsbr0")
 
 
           with subtest("Verify openvswitch bridge"):
-              machine.succeed("ovs-vsctl br-exists ovsbr0")
+              server.succeed("ovs-vsctl br-exists ovsbr0")
         ''
 
     +
       lib.optionalString cfg.storage.zfs # python
         ''
           with subtest("Verify zfs pool created and usable"):
-              machine.succeed(
+              server.succeed(
                   "zpool status",
                   "parted --script /dev/vdb mklabel gpt",
                   "zpool create zfs_pool /dev/vdb",
               )
 
-              machine.succeed("incus storage create zfs_pool zfs source=zfs_pool/incus")
-              machine.succeed("zfs list zfs_pool/incus")
+              server.succeed("incus storage create zfs_pool zfs source=zfs_pool/incus")
+              server.succeed("zfs list zfs_pool/incus")
 
-              machine.succeed("incus storage volume create zfs_pool test_fs --type filesystem")
-              machine.succeed("incus storage volume create zfs_pool test_vol --type block")
+              server.succeed("incus storage volume create zfs_pool test_fs --type filesystem")
+              server.succeed("incus storage volume create zfs_pool test_vol --type block")
 
-              machine.succeed("incus storage show zfs_pool")
-              machine.succeed("incus storage volume list zfs_pool")
-              machine.succeed("incus storage volume show zfs_pool test_fs")
-              machine.succeed("incus storage volume show zfs_pool test_vol")
+              server.succeed("incus storage show zfs_pool")
+              server.succeed("incus storage volume list zfs_pool")
+              server.succeed("incus storage volume show zfs_pool test_fs")
+              server.succeed("incus storage volume show zfs_pool test_vol")
 
-              machine.succeed("incus create zfs1 --empty --storage zfs_pool")
-              machine.succeed("incus list zfs1")
+              server.succeed("incus create zfs1 --empty --storage zfs_pool")
+              server.succeed("incus list zfs1")
         ''
 
     +
       lib.optionalString cfg.storage.lvm # python
         ''
           with subtest("Verify lvm pool created and usable"):
-              machine.succeed("incus storage create lvm_pool lvm source=/dev/vdc lvm.vg_name=incus_pool")
-              machine.succeed("vgs incus_pool")
+              server.succeed("incus storage create lvm_pool lvm source=/dev/vdc lvm.vg_name=incus_pool")
+              server.succeed("vgs incus_pool")
 
-              machine.succeed("incus storage volume create lvm_pool test_fs --type filesystem")
-              machine.succeed("incus storage volume create lvm_pool test_vol --type block")
+              server.succeed("incus storage volume create lvm_pool test_fs --type filesystem")
+              server.succeed("incus storage volume create lvm_pool test_vol --type block")
 
-              machine.succeed("incus storage show lvm_pool")
+              server.succeed("incus storage show lvm_pool")
 
-              machine.succeed("incus storage volume list lvm_pool")
-              machine.succeed("incus storage volume show lvm_pool test_fs")
-              machine.succeed("incus storage volume show lvm_pool test_vol")
+              server.succeed("incus storage volume list lvm_pool")
+              server.succeed("incus storage volume show lvm_pool test_fs")
+              server.succeed("incus storage volume show lvm_pool test_vol")
 
-              machine.succeed("incus create lvm1 --empty --storage lvm_pool")
-              machine.succeed("incus list lvm1")
-        '';
+              server.succeed("incus create lvm1 --empty --storage lvm_pool")
+              server.succeed("incus list lvm1")
+        ''
+    +
+      lib.optionalString cfg.appArmor # python
+        ''
+          with subtest("Verify AppArmor service is started without issue"):
+              # restart AppArmor service since the Incus AppArmor folders are
+              # created after AA service is started
+              server.systemctl("restart apparmor.service")
+              server.succeed("systemctl --no-pager -l status apparmor.service")
+              server.wait_for_unit("apparmor.service")
+        ''
+    +
+      lib.optionalString cfg.feature.user # python
+        ''
+          with subtest("incus-user allows restricted access for users"):
+              server.fail("incus project show user-1000")
+              server.succeed("su - testuser bash -c 'incus list'")
+              # a project is created dynamically for the user
+              server.succeed("incus project show user-1000")
+              # users shouldn't be able to list storage pools
+              server.fail("su - testuser bash -c 'incus storage list'")
+
+
+        ''
+    + instanceScript;
 }

nixos/tests/incus/incus_machine.py

@@ -87,6 +87,6 @@ class IncusHost(Machine):
             .strip()
             .split(" ")[-1]
         )
-        assert (
-            "1" == sysctl
-        ), f"systemd-sysctl configuration not correctly applied, {sysctl} != 1"
+        assert "1" == sysctl, (
+            f"systemd-sysctl configuration not correctly applied, {sysctl} != 1"
+        )

nixos/tests/incus/ui.nix

@@ -1,7 +1,7 @@
 {
   pkgs,
   lib,
-  lts ? true,
+  package,
   ...
 }:
 {
@@ -17,7 +17,7 @@
 
       virtualisation.incus = {
         enable = true;
-        package = if lts then pkgs.incus-lts else pkgs.incus;
+        inherit package;
 
         preseed.config."core.https_address" = ":8443";
         ui.enable = true;