e57inspector.tests: Add VLM screenshot analysis for automatic test checking

On an AMD Ryzen 9 5950X I got:

    VLM inference took 29.9s

Smaller LLMs are faster, but the current (Gemma4 E2B Q4)
is more reliable in its outputs and can correctly describe
what's on screen.

Allowing more than `--threads 1` makes it much faster,
but at the potential cost of nondeterministic
output of `llama-cli`.

The VLM screenshot analysis running in CI makes it slower,
but spending < 1 minute single-core seems worth it
given that it quite robustly removes the need for a human
to check if the GUI renders correctly.

Potential future improvement:
A bit benefit would be content-addressing of the screenshot:
Most nixpkgs update would likely not change the screenshot
pixels at all, reproducing it bit-identically.
In that case, there's no need to re-run the VLM at all.
Doing the VLM analysis in a separate derivation that takes
the screenshot as only input would make it ossible
to cache these results, while still being less maintenance
than having a "golden" screenshot that needs to be updated
by a human when screenshots actually change.

nixos/tests/all-tests.nix

@@ -495,7 +495,7 @@ in
   drupal = runTest ./drupal.nix;
   dublin-traceroute = runTest ./dublin-traceroute.nix;
   dwl = runTestOn [ "x86_64-linux" "aarch64-linux" ] ./dwl.nix;
-  e57inspector = runTest ./e57inspector.nix;
+  e57inspector = import ./e57inspector.nix { inherit pkgs runTest; };
   early-mount-options = runTest ./early-mount-options.nix;
   earlyoom = runTestOn [ "x86_64-linux" ] ./earlyoom.nix;
   easytier = runTest ./easytier.nix;

nixos/tests/e57inspector.nix

@@ -1,38 +1,53 @@
-{ pkgs, ... }:
-{
-  name = "e57inspector";
-  meta.maintainers = with pkgs.lib.maintainers; [
-    nh2
-    chpatrick
-  ];
+{ pkgs, runTest }:
+let
+  testFile = pkgs.fetchurl {
+    url = "https://raw.githubusercontent.com/asmaloney/libE57Format-test-data/bbcacec05d60f923869545c5eab33d94c390d50e/self/ColouredCubeFloat.e57";
+    hash = "sha256-bb95crNYvX3Qhkx4k6Sqe2GjOf1u4nxxswMfdjyXfTM=";
+  };
 
-  nodes.machine =
-    { ... }:
-    {
-      imports = [
-        ./common/x11.nix
-      ];
+  vmTest = runTest {
+    name = "e57inspector-vm";
+    meta.maintainers = with pkgs.lib.maintainers; [
+      nh2
+      chpatrick
+    ];
 
-      services.xserver.enable = true;
-      environment.systemPackages = [
-        pkgs.e57inspector
-        pkgs.xdotool
-      ];
-    };
+    nodes.machine =
+      { ... }:
+      {
+        imports = [
+          ./common/x11.nix
+        ];
 
-  testScript =
-    let
-      testFile = pkgs.fetchurl {
-        url = "https://raw.githubusercontent.com/asmaloney/libE57Format-test-data/bbcacec05d60f923869545c5eab33d94c390d50e/self/ColouredCubeFloat.e57";
-        hash = "sha256-bb95crNYvX3Qhkx4k6Sqe2GjOf1u4nxxswMfdjyXfTM=";
+        services.xserver.enable = true;
+        environment.systemPackages = [
+          pkgs.e57inspector
+        ];
       };
-    in
-    ''
+    enableOCR = true;
+
+    testScript = ''
       start_all()
       machine.wait_for_x()
 
       machine.execute("e57inspector ${testFile} >&2 &")
-      machine.wait_until_succeeds("xdotool search --pid $(pidof .e57inspector-wrapped)")
-      machine.screenshot("screen")
+      machine.wait_for_text("File")  # menu visible
+
+      machine.screenshot("screen.png")
     '';
+  };
+in
+{
+  vm = vmTest;
+
+  screenshot-analysis = pkgs.callPackage ./vlm-screenshot-question.nix {
+    name = "e57inspector-screenshot-analysis";
+    screenshot = "${vmTest}/screen.png";
+    question = ''
+      Look at this screenshot of a desktop application.
+      Answer: Does the application show that a file was loaded into it successfully?
+      For this, only scans matter, as there are no images in the file.
+      The inspector on the left should show child elements below 'Data 3D'.
+    '';
+  };
 }

nixos/tests/vlm-screenshot-question.nix

@@ -0,0 +1,118 @@
+# Reusable VLM screenshot analysis derivation.
+#
+# Similar to `wait_for_text()` in NixOS VM tests.
+#
+# Runs a VLM (Vision Language Model) on a screenshot and asserts that the
+# model's answer to a yes/no question ends with "YES".
+#
+# This is useful to automatically test software that is otherwise
+# hard to test, e.g. "does this 3D program render the bunny correctly?".
+# It is especially useful to judge screenshots made in NixOS VM tests.
+{
+  lib,
+  writers,
+  fetchurl,
+  llama-cpp,
+  runCommand,
+  # VLM defaults, chosen to pick a model smart enough to be useful
+  # for screenshot analysis, but small enough to not consume too much RAM
+  # or be too slow for CI.
+  model ? (
+    fetchurl {
+      url = "https://huggingface.co/unsloth/gemma-4-E2B-it-GGUF/resolve/90f9618340396838ee7ff5b0ba2da27da62953d3/gemma-4-E2B-it-Q4_0.gguf";
+      hash = "sha256-nEwdSKRi9/iDsomErE9C02bJxXNDTqtoVT4POL9+tQw=";
+    }
+  ),
+  mmproj ? (
+    fetchurl {
+      url = "https://huggingface.co/unsloth/gemma-4-E2B-it-GGUF/resolve/90f9618340396838ee7ff5b0ba2da27da62953d3/mmproj-F16.gguf";
+      hash = "sha256-FAvo14SXQfiMUHV9UpuENz7o4nBSzCI2hVtTf0qCFfo=";
+    }
+  ),
+  # User-provided arguments:
+  name,
+  screenshot,
+  question,
+}:
+let
+
+  analysisScript =
+    writers.writePython3 "${name}-script" { flakeIgnore = [ "E501" ]; } # allow long lines
+      ''
+        import os
+        import re
+        import subprocess
+        import time
+
+        out = os.environ["out"]
+        screenshot = "${screenshot}"
+        # Using JSON here even permits preserving multi-line ASCII art questions and so on.
+        question = ${builtins.toJSON question}
+
+        # Build the full prompt with output markers for reliable extraction.
+        prompt = (
+            "Start your output with [output-start]."
+            f" {question}"
+            " Explain what you see, and your judgment."
+            " Then answer that question with exactly YES or NO, followed by [output-end]."
+        )
+
+        vlm_start = time.time()
+        result = subprocess.run(
+            [
+                "${lib.getExe llama-cpp}",
+                "--single-turn", "--no-display-prompt", "--log-verbosity", "0", "--jinja",
+                "--simple-io",  # disables the spinner whose backspace chars would corrupt captured output
+                "--reasoning", "off", "--temp", "0",
+                "--threads", "1",  # for determinism
+                "--n-gpu-layers", "0",  # force CPU-only (results on GPUs might be different and nondeterministic, see https://github.com/ggml-org/llama.cpp/pull/16016#issuecomment-3293505238)
+                "--model", "${model}",
+                "--mmproj", "${mmproj}",
+                "--image", screenshot,
+                "-p", prompt,
+            ],
+            capture_output=True,
+            text=True,
+            # `OMP_NUM_THREADS=1` prevents OpenMP from spawning extra threads in the BLAS backend
+            # (OpenBLAS), which causes nondeterminism with `--image`; without `--image`, `--threads 1`
+            # alone is already deterministic (BLAS is not used for short text prompts).
+            # Relevant code: https://github.com/ggml-org/llama.cpp/blob/80afa33aadcc4f71212b17e5e52904491c76b63e/ggml/src/ggml-blas/ggml-blas.cpp#L30-L148
+            # PR to fix it in OpenBLAS: https://github.com/OpenMathLib/OpenBLAS/pull/5808
+            env={**os.environ, "OMP_NUM_THREADS": "1"},
+        )
+        vlm_elapsed = time.time() - vlm_start
+        output = result.stdout
+        print(f"VLM inference took {vlm_elapsed:.1f}s")
+        print(f"VLM raw output: {repr(output)}")
+        if result.returncode != 0:
+            print(f"VLM stderr: {result.stderr}")
+        assert result.returncode == 0, f"llama-cli failed with exit code {result.returncode}"
+
+        print()
+
+        # Post-process: extract the answer between `[output-start]` and `[output-end]` markers.
+        # This is needed because llama-cli prints UI noise (banner,
+        # spinner, stats) to stdout alongside the model's response.
+        # TODO: Replace with `--quiet` once https://github.com/ggml-org/llama.cpp/pull/22848 is merged;
+        #       then also remove the markers from the prompt and the extraction below.
+        matches = re.findall(r"\[output-start\](.*?)\[output-end\]", output, re.DOTALL)
+        assert matches, (
+            f"VLM output did not contain [output-start]...[output-end] markers."
+            f" Raw output: {repr(output)}"
+        )
+        answer = matches[-1].strip()  # use last match (first may be prompt echo)
+        print("VLM answer:")
+        print(answer)
+        assert answer.upper().endswith("YES"), (
+            f"VLM did not confirm expected answer. Answer: {answer}"
+        )
+
+        os.makedirs(out, exist_ok=True)
+        with open(os.path.join(out, "vlm-answer.txt"), "w") as f:
+            f.write(answer + "\n")
+        os.symlink(screenshot, os.path.join(out, "screen.png"))
+      '';
+in
+runCommand name { } ''
+  ${analysisScript}
+''

pkgs/by-name/e5/e57inspector/package.nix

@@ -62,9 +62,7 @@ stdenv.mkDerivation (finalAttrs: {
     runHook postInstall
   '';
 
-  passthru.tests = {
-    e57inspector = nixosTests.e57inspector;
-  };
+  passthru.tests = nixosTests.e57inspector;
 
   meta = {
     description = "Cross-platform E57 file viewer to list and view stored point clouds, images and metadata";