{
  runTest,
  lib,
}:

let
  /*
    * The attrset `exporterTests` contains one attribute
    * for each exporter test. Each of these attributes
    * is expected to be an attrset containing:
    *
    *  `exporterConfig`:
    *    this attribute set contains config for the exporter itself
    *
    *  `exporterTest`
    *    this attribute set contains test instructions
    *
    *  `metricProvider` (optional)
    *    this attribute contains additional machine config
    *
    *  `nodeName` (optional)
    *    override an incompatible testnode name
    *
    *  Example:
    *    exporterTests.<exporterName> = {
    *      exporterConfig = {
    *        enable = true;
    *      };
    *      metricProvider = {
    *        services.<metricProvider>.enable = true;
    *      };
    *      exporterTest = ''
    *        wait_for_unit("prometheus-<exporterName>-exporter.service")
    *        wait_for_open_port(1234)
    *        succeed("curl -sSf 'localhost:1234/metrics'")
    *      '';
    *    };
    *
    *  # this would generate the following test config:
    *
    *    nodes.<exporterName> = {
    *      services.prometheus.<exporterName> = {
    *        enable = true;
    *      };
    *      services.<metricProvider>.enable = true;
    *    };
    *
    *    testScript = ''
    *      <exporterName>.start()
    *      <exporterName>.wait_for_unit("prometheus-<exporterName>-exporter.service")
    *      <exporterName>.wait_for_open_port(1234)
    *      <exporterName>.succeed("curl -sSf 'localhost:1234/metrics'")
    *      <exporterName>.shutdown()
    *    '';
  */

  exporterTests = {
    apcupsd =
      { ... }:
      {
        exporterConfig = {
          enable = true;
        };
        metricProvider = {
          services.apcupsd.enable = true;
        };
        exporterTest = ''
          wait_for_unit("apcupsd.service")
          wait_for_open_port(3551)
          wait_for_unit("prometheus-apcupsd-exporter.service")
          wait_for_open_port(9162)
          succeed("curl -sSf http://localhost:9162/metrics | grep 'apcupsd_info'")
        '';
      };

    artifactory =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          artiUsername = "artifactory-username";
          artiPassword = "artifactory-password";
        };
        exporterTest = ''
          wait_for_unit("prometheus-artifactory-exporter.service")
          wait_for_open_port(9531)
          succeed(
              "curl -sSf http://localhost:9531/metrics | grep 'artifactory_up'"
          )
        '';
      };

    bind =
      { ... }:
      {
        exporterConfig = {
          enable = true;
        };
        metricProvider = {
          services.bind.enable = true;
          services.bind.extraConfig = ''
            statistics-channels {
              inet 127.0.0.1 port 8053 allow { localhost; };
            };
          '';
        };
        exporterTest = ''
          wait_for_unit("prometheus-bind-exporter.service")
          wait_for_open_port(9119)
          succeed(
              "curl -sSf http://localhost:9119/metrics | grep 'bind_query_recursions_total 0'"
          )
        '';
      };

    bird =
      { ... }:
      {
        exporterConfig = {
          enable = true;
        };
        metricProvider = {
          services.bird.enable = true;
          services.bird.config = ''
            router id 127.0.0.1;

            protocol kernel MyObviousTestString {
              ipv4 {
                import all;
                export none;
              };
            }

            protocol device {
            }
          '';
        };
        exporterTest = ''
          wait_for_unit("prometheus-bird-exporter.service")
          wait_for_open_port(9324)
          wait_until_succeeds(
              "curl -sSf http://localhost:9324/metrics | grep 'MyObviousTestString'"
          )
        '';
      };

    bitcoin =
      { pkgs, ... }:
      {
        exporterConfig = {
          enable = true;
          rpcUser = "bitcoinrpc";
          rpcPasswordFile = pkgs.writeText "password" "hunter2";
        };
        metricProvider = {
          services.bitcoind.default.enable = true;
          services.bitcoind.default.rpc.users.bitcoinrpc.passwordHMAC =
            "e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7";
        };
        exporterTest = ''
          wait_for_unit("prometheus-bitcoin-exporter.service")
          wait_for_unit("bitcoind-default.service")
          wait_for_open_port(9332)
          succeed("curl -sSf http://localhost:9332/metrics | grep '^bitcoin_blocks '")
        '';
      };

    blackbox =
      { pkgs, ... }:
      {
        exporterConfig = {
          enable = true;
          configFile = pkgs.writeText "config.yml" (
            builtins.toJSON {
              modules.icmp_v6 = {
                prober = "icmp";
                icmp.preferred_ip_protocol = "ip6";
              };
            }
          );
        };
        exporterTest = ''
          wait_for_unit("prometheus-blackbox-exporter.service")
          wait_for_open_port(9115)
          succeed(
              "curl -sSf 'http://localhost:9115/probe?target=localhost&module=icmp_v6' | grep 'probe_success 1'"
          )
        '';
      };

    borgmatic =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          user = "root";
        };
        metricProvider = {
          services.borgmatic.enable = true;
          services.borgmatic.settings.source_directories = [ "/home" ];
          services.borgmatic.settings.repositories = [
            {
              label = "local";
              path = "/var/backup";
            }
          ];
          services.borgmatic.settings.keep_daily = 10;
        };
        exporterTest = ''
          succeed("borgmatic rcreate -e none")
          succeed("borgmatic")
          wait_for_unit("prometheus-borgmatic-exporter.service")
          wait_for_open_port(9996)
          succeed("curl -sSf localhost:9996/metrics | grep 'borg_total_backups{repository=\"/var/backup\"} 1'")
        '';
      };

    collectd =
      { lib, ... }:
      {
        exporterConfig = {
          enable = true;
          extraFlags = [ "--web.collectd-push-path /collectd" ];
        };
        exporterTest =
          let
            postData = lib.replaceStrings [ "\n" ] [ "" ] ''
              [{
                "values":[23],
                "dstypes":["gauge"],
                "type":"gauge",
                "interval":1000,
                "host":"testhost",
                "plugin":"testplugin",
                "time":DATE
              }]
            '';
          in
          ''
            wait_for_unit("prometheus-collectd-exporter.service")
            wait_for_open_port(9103)
            succeed(
                'echo \'${postData}\'> /tmp/data.json'
            )
            succeed('sed -i -e "s DATE $(date +%s) " /tmp/data.json')
            succeed(
                "curl -sSfH 'Content-Type: application/json' -X POST --data @/tmp/data.json localhost:9103/collectd"
            )
            succeed(
                "curl -sSf localhost:9103/metrics | grep 'collectd_testplugin_gauge{instance=\"testhost\"} 23'"
            )
          '';
      };

    deluge =
      { pkgs, ... }:
      {
        exporterConfig = {
          enable = true;
          port = 1234;
          listenAddress = "127.0.0.1";

          delugeUser = "user";
          delugePort = 2345;
          delugePasswordFile = pkgs.writeText "password" "weak_password";
        };
        metricProvider = {
          services.deluge.enable = true;
          services.deluge.declarative = true;
          services.deluge.config.daemon_port = 2345;
          services.deluge.authFile = pkgs.writeText "authFile" ''
            localclient:abcdef:10
            user:weak_password:10
          '';
        };
        exporterTest = ''
          wait_for_unit("deluged.service")
          wait_for_open_port(2345)
          wait_for_unit("prometheus-deluge-exporter.service")
          wait_for_open_port(1234)
          succeed("curl -sSf http://localhost:1234 | grep 'deluge_torrents'")
        '';
      };

    dnsmasq =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          leasesPath = "/var/lib/dnsmasq/dnsmasq.leases";
        };
        metricProvider = {
          services.dnsmasq.enable = true;
        };
        exporterTest = ''
          wait_for_unit("dnsmasq.service")
          wait_for_open_port(53)
          wait_for_file("/var/lib/dnsmasq/dnsmasq.leases")
          wait_for_unit("prometheus-dnsmasq-exporter.service")
          wait_for_open_port(9153)
          succeed("curl -sSf http://localhost:9153/metrics | grep 'dnsmasq_leases 0'")
        '';
      };

    dnssec =
      { pkgs, ... }:
      {
        exporterConfig = {
          enable = true;
          configuration = {
            records = [
              {
                zone = "example.com";
                record = "@";
                type = "SOA";
              }
            ];
          };
          resolvers = [ "127.0.0.1:53" ];
        };
        metricProvider = {
          services.knot = {
            enable = true;
            settingsFile = pkgs.writeText "knot.conf" ''
              server:
                listen: 127.0.0.1@53
              template:
                - id: default
                  storage: ${
                    pkgs.buildEnv {
                      name = "zones";
                      paths = [
                        (pkgs.writeTextDir "example.com.zone" ''
                          @ SOA ns1.example.com. noc.example.com. 2024032401 86400 7200 3600000 172800
                          @       NS      ns1
                          ns1     A       192.168.0.1
                        '')
                      ];
                    }
                  }
                  zonefile-load: difference
                  zonefile-sync: -1
              zone:
                - domain: example.com
                  file: example.com.zone
                  dnssec-signing: on
            '';
          };
        };
        exporterTest = ''
          wait_for_unit("knot.service")
          wait_for_open_port(53)
          wait_for_unit("prometheus-dnssec-exporter.service")
          wait_for_open_port(9204)
          succeed("curl -sSf http://localhost:9204/metrics | grep 'example.com'")
        '';
      };

    # Access to WHOIS server is required to properly test this exporter, so
    # just perform basic sanity check that the exporter is running and returns
    # a failure.
    domain =
      { ... }:
      {
        exporterConfig = {
          enable = true;
        };
        exporterTest = ''
          wait_for_unit("prometheus-domain-exporter.service")
          wait_for_open_port(9222)
          succeed("curl -sSf 'http://localhost:9222/probe?target=nixos.org'")
        '';
      };

    dovecot =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          scopes = [ "global" ];
          socketPath = "/var/run/dovecot2/old-stats";
          user = "root"; # <- don't use user root in production
        };
        metricProvider = {
          services.dovecot2.enable = true;
        };
        exporterTest = ''
          wait_for_unit("prometheus-dovecot-exporter.service")
          wait_for_open_port(9166)
          succeed(
              "curl -sSf http://localhost:9166/metrics | grep 'dovecot_up{scope=\"global\"} 1'"
          )
        '';
      };

    exportarr-sonarr =
      { pkgs, ... }:
      let
        apikey = "eccff6a992bc2e4b88e46d064b26bb4e";
      in
      {
        nodeName = "exportarr_sonarr";
        exporterConfig = {
          enable = true;
          url = "http://127.0.0.1:8989";
          apiKeyFile = pkgs.writeText "dummy-api-key" apikey;
        };
        metricProvider = {
          services.sonarr = {
            enable = true;
            environmentFiles = [ (pkgs.writeText "sonarr-env" "SONARR__AUTH__APIKEY=${apikey}") ];
          };
        };
        exporterTest = ''
          wait_for_unit("sonarr.service")
          wait_for_open_port(8989)
          wait_for_unit("prometheus-exportarr-sonarr-exporter.service")
          wait_for_open_port(9708)
          succeed("curl -sSf http://localhost:9708/metrics | grep sonarr_series_total")
        '';
      };

    ebpf =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          names = [ "timers" ];
        };
        exporterTest = ''
          wait_for_unit("prometheus-ebpf-exporter.service")
          wait_for_open_port(9435)
          succeed(
              "curl -sSf http://localhost:9435/metrics | grep 'ebpf_exporter_enabled_configs{name=\"timers\"} 1'"
          )
        '';
      };

    elasticsearch =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          url = "http://localhost:9200";
        };
        metricProvider = {
          # `services.elasticsearch` is unmaintained; OpenSearch is the same
          # engine class and is explicitly supported by the exporter.
          services.opensearch.enable = true;
          virtualisation.memorySize = 2048;
        };
        exporterTest = ''
          wait_for_unit("opensearch.service")
          wait_for_open_port(9200)
          wait_for_unit("prometheus-elasticsearch-exporter.service")
          wait_for_open_port(9114)
          succeed(
              "curl -sSf localhost:9114/metrics | grep 'elasticsearch_cluster_health_status'"
          )
        '';
      };

    fail2ban =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          exitOnError = true;
        };
        metricProvider = {
          services.fail2ban.enable = true;
        };
        exporterTest = ''
          wait_for_unit("fail2ban.service")
          wait_for_unit("prometheus-fail2ban-exporter.service")
          wait_for_open_port(9191)
          succeed("curl -sSf http://localhost:9191/metrics | grep 'f2b_errors'")
        '';
      };

    fastly =
      { pkgs, ... }:
      {
        exporterConfig = {
          enable = true;
          environmentFile = pkgs.writeText "fastly-exporter-env" "FASTLY_API_TOKEN=abc123";
        };

        exporterTest = ''
          wait_for_unit("prometheus-fastly-exporter.service")
          wait_for_open_port(9118)
        '';
      };

    fritzbox =
      { ... }:
      {
        # TODO add proper test case
        exporterConfig = {
          enable = true;
        };
        exporterTest = ''
          wait_for_unit("prometheus-fritzbox-exporter.service")
          wait_for_open_port(9133)
          succeed(
              "curl -sSf http://localhost:9133/metrics | grep 'fritzbox_exporter_collect_errors 0'"
          )
        '';
      };

    graphite =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          port = 9108;
          graphitePort = 9109;
          mappingSettings.mappings = [
            {
              match = "test.*.*";
              name = "testing";
              labels = {
                protocol = "$1";
                author = "$2";
              };
            }
          ];
        };
        exporterTest = ''
          wait_for_unit("prometheus-graphite-exporter.service")
          wait_for_open_port(9108)
          wait_for_open_port(9109)
          succeed("echo test.tcp.foo-bar 1234 $(date +%s) | nc -w1 localhost 9109")
          succeed("curl -sSf http://localhost:9108/metrics | grep 'testing{author=\"foo-bar\",protocol=\"tcp\"} 1234'")
        '';
      };

    idrac =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          port = 9348;
          configuration = {
            hosts = {
              default = {
                username = "username";
                password = "password";
              };
            };
          };
        };
        exporterTest = ''
          wait_for_unit("prometheus-idrac-exporter.service")
          wait_for_open_port(9348)
          wait_until_succeeds("curl localhost:9348")
        '';
      };

    influxdb =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          sampleExpiry = "3s";
        };
        exporterTest = ''
          wait_for_unit("prometheus-influxdb-exporter.service")
          wait_for_open_port(9122)
          succeed(
            "curl -XPOST http://localhost:9122/write --data-binary 'influxdb_exporter,distro=nixos,added_in=21.09 value=1'"
          )
          succeed(
            "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
          )
          execute("sleep 5")
          fail(
            "curl -sSf http://localhost:9122/metrics | grep 'nixos'"
          )
        '';
      };

    ipmi =
      { ... }:
      {
        exporterConfig = {
          enable = true;
        };
        exporterTest = ''
          wait_for_unit("prometheus-ipmi-exporter.service")
          wait_for_open_port(9290)
          succeed(
            "curl -sSf http://localhost:9290/metrics | grep 'ipmi_scrape_duration_seconds'"
          )
        '';
      };

    jitsi =
      { ... }:
      {
        exporterConfig = {
          enable = true;
        };
        metricProvider = {
          systemd.services.prometheus-jitsi-exporter.after = [ "jitsi-videobridge2.service" ];
          services.jitsi-videobridge = {
            enable = true;
            colibriRestApi = true;
          };
        };
        exporterTest = ''
          wait_for_unit("jitsi-videobridge2.service")
          wait_for_open_port(8080)
          wait_for_unit("prometheus-jitsi-exporter.service")
          wait_for_open_port(9700)
          wait_until_succeeds(
              'journalctl -eu prometheus-jitsi-exporter.service -o cat | grep "key=participants"'
          )
          succeed("curl -sSf 'localhost:9700/metrics' | grep 'jitsi_participants 0'")
        '';
      };

    json =
      { pkgs, ... }:
      {
        exporterConfig = {
          enable = true;
          configFile = pkgs.writeText "json-exporter-conf.json" (
            builtins.toJSON {
              modules = {
                default = {
                  metrics = [
                    {
                      name = "json_test_metric";
                      path = "{ .test }";
                    }
                  ];
                };
              };
            }
          );
        };
        metricProvider = {
          systemd.services.prometheus-json-exporter.after = [ "nginx.service" ];
          services.nginx = {
            enable = true;
            virtualHosts.localhost.locations."/".extraConfig = ''
              return 200 "{\"test\":1}";
            '';
          };
        };
        exporterTest = ''
          wait_for_unit("nginx.service")
          wait_for_open_port(80)
          wait_for_unit("prometheus-json-exporter.service")
          wait_for_open_port(7979)
          succeed(
              "curl -sSf 'localhost:7979/probe?target=http://localhost' | grep 'json_test_metric 1'"
          )
        '';
      };

    kafka =
      { pkgs, ... }:
      {
        exporterConfig = {
          enable = true;
          environmentFile = pkgs.writeTextFile {
            name = "/tmp/prometheus-kafka-exporter.env";
            text = ''
              KAFKA_BROKERS="localhost:9092"
            '';
          };
        };
        metricProvider = {
          services.apache-kafka = {
            enable = true;

            clusterId = "pHG8aWuXSfWAibHFDCnsCQ";

            formatLogDirs = true;

            settings = {
              "node.id" = 1;
              "process.roles" = [
                "broker"
                "controller"
              ];
              "listeners" = [
                "PLAINTEXT://:9092"
                "CONTROLLER://:9093"
              ];
              "listener.security.protocol.map" = [
                "PLAINTEXT:PLAINTEXT"
                "CONTROLLER:PLAINTEXT"
              ];
              "controller.quorum.voters" = [
                "1@localhost:9093"
              ];
              "controller.listener.names" = [ "CONTROLLER" ];
              "log.dirs" = [ "/var/lib/apache-kafka" ];
            };
          };

          systemd.services.apache-kafka.serviceConfig.StateDirectory = "apache-kafka";
        };
        exporterTest = ''
          wait_for_unit("apache-kafka")
          wait_for_open_port(9092)
          wait_for_open_port(9093)
          wait_for_unit("prometheus-kafka-exporter.service")
          wait_for_open_port(8080)
          wait_until_succeeds(
            "journalctl -o cat -u prometheus-kafka-exporter.service | grep '\"version\":\"${pkgs.kminion.version}\"'"
          )
          succeed(
              "curl -sSf http://localhost:8080/metrics | grep 'kminion_exporter_up'"
          )
        '';
      };

    knot =
      { pkgs, ... }:
      {
        exporterConfig = {
          enable = true;
        };
        metricProvider = {
          services.knot = {
            enable = true;
            extraArgs = [ "-v" ];
            settingsFile = pkgs.writeText "knot.conf" ''
              server:
                listen: 127.0.0.1@53

              template:
                - id: default
                  global-module: mod-stats
                  dnssec-signing: off
                  zonefile-sync: -1
                  zonefile-load: difference
                  storage: ${
                    pkgs.buildEnv {
                      name = "foo";
                      paths = [
                        (pkgs.writeTextDir "test.zone" ''
                          @ SOA ns.example.com. noc.example.com. 2019031301 86400 7200 3600000 172800
                          @       NS      ns1
                          @       NS      ns2
                          ns1     A       192.168.0.1
                        '')
                      ];
                    }
                  }

              mod-stats:
                - id: custom
                  edns-presence: on
                  query-type: on

              zone:
                - domain: test
                  file: test.zone
                  module: mod-stats/custom
            '';
          };
        };
        exporterTest = ''
          wait_for_unit("knot.service")
          wait_for_unit("prometheus-knot-exporter.service")
          wait_for_open_port(9433)
          succeed("curl -sSf 'localhost:9433' | grep '2\.019031301'")
        '';
      };

    keylight =
      { ... }:
      {
        # A hardware device is required to properly test this exporter, so just
        # perform a couple of basic sanity checks that the exporter is running
        # and requires a target, but cannot reach a specified target.
        exporterConfig = {
          enable = true;
        };
        exporterTest = ''
          wait_for_unit("prometheus-keylight-exporter.service")
          wait_for_open_port(9288)
          succeed(
              "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics | grep '400'"
          )
          succeed(
              "curl -sS --write-out '%{http_code}' -o /dev/null http://localhost:9288/metrics?target=nosuchdevice | grep '500'"
          )
        '';
      };

    lnd =
      { pkgs, ... }:
      {
        exporterConfig = {
          enable = true;
          lndTlsPath = "/var/lib/lnd/tls.cert";
          lndMacaroonDir = "/var/lib/lnd";
          extraFlags = [ "--lnd.network=regtest" ];
        };
        metricProvider = {
          systemd.services.prometheus-lnd-exporter.serviceConfig.RestartSec = 15;
          systemd.services.prometheus-lnd-exporter.after = [ "lnd.service" ];
          services.bitcoind.regtest = {
            enable = true;
            extraConfig = ''
              rpcauth=bitcoinrpc:e8fe33f797e698ac258c16c8d7aadfbe$872bdb8f4d787367c26bcfd75e6c23c4f19d44a69f5d1ad329e5adf3f82710f7
              zmqpubrawblock=tcp://127.0.0.1:28332
              zmqpubrawtx=tcp://127.0.0.1:28333
              # https://github.com/lightningnetwork/lnd/issues/9163
              deprecatedrpc=warnings
            '';
            extraCmdlineOptions = [ "-regtest" ];
          };
          systemd.services.lnd = {
            serviceConfig.ExecStart = ''
              ${pkgs.lnd}/bin/lnd \
                --datadir=/var/lib/lnd \
                --tlscertpath=/var/lib/lnd/tls.cert \
                --tlskeypath=/var/lib/lnd/tls.key \
                --logdir=/var/log/lnd \
                --bitcoin.active \
                --bitcoin.regtest \
                --bitcoin.node=bitcoind \
                --bitcoind.rpcuser=bitcoinrpc \
                --bitcoind.rpcpass=hunter2 \
                --bitcoind.zmqpubrawblock=tcp://127.0.0.1:28332 \
                --bitcoind.zmqpubrawtx=tcp://127.0.0.1:28333 \
                --readonlymacaroonpath=/var/lib/lnd/readonly.macaroon
            '';
            serviceConfig.StateDirectory = "lnd";
            wantedBy = [ "multi-user.target" ];
            after = [ "network.target" ];
          };
          # initialize wallet, creates macaroon needed by exporter
          systemd.services.lnd.postStart = ''
            until [ -f /var/lib/lnd/tls.cert ]; do sleep 1; done
            ${pkgs.curl}/bin/curl \
              --retry 20 \
              --retry-delay 1 \
              --retry-connrefused \
              --cacert /var/lib/lnd/tls.cert \
              -X GET \
              https://localhost:8080/v1/genseed | ${pkgs.jq}/bin/jq -c '.cipher_seed_mnemonic' > /tmp/seed
            ${pkgs.curl}/bin/curl \
              --retry 20 \
              --retry-delay 1 \
              --retry-connrefused \
              --cacert /var/lib/lnd/tls.cert \
              -X POST \
              -d "{\"wallet_password\": \"asdfasdfasdf\", \"cipher_seed_mnemonic\": $(cat /tmp/seed | tr -d '\n')}" \
              https://localhost:8080/v1/initwallet
          '';
        };
        exporterTest = ''
          wait_for_unit("lnd.service")
          wait_for_open_port(10009)
          wait_for_unit("prometheus-lnd-exporter.service")
          wait_for_open_port(9092)
          succeed("curl -sSf localhost:9092/metrics | grep '^lnd_peer_count'")
        '';
      };

    mail =
      { pkgs, ... }:
      {
        exporterConfig = {
          enable = true;
          configuration = {
            monitoringInterval = "2s";
            mailCheckTimeout = "10s";
            servers = [
              {
                name = "testserver";
                server = "localhost";
                port = 25;
                from = "mail-exporter@localhost";
                to = "mail-exporter@localhost";
                detectionDir = "/var/spool/mail/mail-exporter/new";
              }
            ];
          };
        };
        metricProvider = {
          services.postfix.enable = true;
          systemd.services.prometheus-mail-exporter = {
            after = [ "postfix.service" ];
            requires = [ "postfix.service" ];
            serviceConfig = {
              ExecStartPre = [
                "${pkgs.writeShellScript "create-maildir" ''
                  mkdir -p -m 0700 mail-exporter/new
                ''}"
              ];
              ProtectHome = true;
              ReadOnlyPaths = "/";
              ReadWritePaths = "/var/spool/mail";
              WorkingDirectory = "/var/spool/mail";
            };
          };
          users.users.mailexporter = {
            isSystemUser = true;
            group = "mailexporter";
          };
          users.groups.mailexporter = { };
        };
        exporterTest = ''
          wait_for_unit("postfix.service")
          wait_for_unit("prometheus-mail-exporter.service")
          wait_for_open_port(9225)
          wait_until_succeeds(
              "curl -sSf http://localhost:9225/metrics | grep 'mail_deliver_success{configname=\"testserver\"} 1'"
          )
        '';
      };

    mikrotik =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          extraFlags = [ "-timeout=1s" ];
          configuration = {
            devices = [
              {
                name = "router";
                address = "192.168.42.48";
                user = "prometheus";
                password = "shh";
              }
            ];
            features = {
              bgp = true;
              dhcp = true;
              dhcpl = true;
              dhcpv6 = true;
              health = true;
              routes = true;
              poe = true;
              pools = true;
              optics = true;
              w60g = true;
              wlansta = true;
              wlanif = true;
              monitor = true;
              ipsec = true;
            };
          };
        };
        exporterTest = ''
          wait_for_unit("prometheus-mikrotik-exporter.service")
          wait_for_open_port(9436)
          succeed(
              "curl -sSf http://localhost:9436/metrics | grep 'mikrotik_scrape_collector_success{device=\"router\"} 0'"
          )
        '';
      };

    modemmanager =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          refreshRate = "10s";
        };
        metricProvider = {
          # ModemManager is installed when NetworkManager is enabled. Ensure it is
          # started and is wanted by NM and the exporter to start everything up
          # in the right order.
          networking.networkmanager.enable = true;
          systemd.services.ModemManager = {
            enable = true;
            wantedBy = [
              "NetworkManager.service"
              "prometheus-modemmanager-exporter.service"
            ];
          };
        };
        exporterTest = ''
          wait_for_unit("ModemManager.service")
          wait_for_unit("prometheus-modemmanager-exporter.service")
          wait_for_open_port(9539)
          succeed(
              "curl -sSf http://localhost:9539/metrics | grep 'modemmanager_info'"
          )
        '';
      };

    mqtt =
      { pkgs, ... }:
      {
        exporterConfig = {
          enable = true;
          environmentFile = pkgs.writeText "mqtt-exporter-envfile" ''
            MQTT_PASSWORD=testpassword
          '';
        };
        metricProvider = {
          services.mosquitto = {
            enable = true;
            listeners = [
              {
                users.exporter = {
                  acl = [ "read #" ];
                  passwordFile = pkgs.writeText "mosquitto-password" "testpassword";
                };
              }
            ];
          };
          systemd.services.prometheus-mqtt-exporter = {
            wants = [ "mosquitto.service" ];
            after = [ "mosquitto.service" ];
          };
        };
        exporterTest = ''
          wait_for_unit("mosquitto.service")
          wait_for_unit("prometheus-mqtt-exporter.service")
          wait_for_open_port(9000)
          succeed(
            "curl -sSf http://localhost:9000/metrics | grep '^python_info'"
          )
        '';
      };

    mysqld =
      { pkgs, ... }:
      {
        exporterConfig = {
          enable = true;
          runAsLocalSuperUser = true;
          configFile = pkgs.writeText "test-prometheus-exporter-mysqld-config.my-cnf" ''
            [client]
            user = exporter
            password = snakeoilpassword
          '';
        };
        metricProvider = {
          services.mysql = {
            enable = true;
            package = pkgs.mariadb;
            initialScript = pkgs.writeText "mysql-init-script.sql" ''
              CREATE USER 'exporter'@'localhost'
              IDENTIFIED BY 'snakeoilpassword'
              WITH MAX_USER_CONNECTIONS 3;
              GRANT PROCESS, REPLICATION CLIENT, SLAVE MONITOR, SELECT ON *.* TO 'exporter'@'localhost';
            '';
          };
        };
        exporterTest = ''
          wait_for_unit("prometheus-mysqld-exporter.service")
          wait_for_open_port(9104)
          wait_for_unit("mysql.service")
          succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 1'")
          systemctl("stop mysql.service")
          succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 0'")
          systemctl("start mysql.service")
          wait_for_unit("mysql.service")
          succeed("curl -sSf http://localhost:9104/metrics | grep 'mysql_up 1'")
        '';
      };

    nextcloud =
      { pkgs, ... }:
      {
        exporterConfig = {
          enable = true;
          passwordFile = "/var/nextcloud-pwfile";
          url = "http://localhost";
        };
        metricProvider = {
          systemd.services.nc-pwfile =
            let
              passfile = (pkgs.writeText "pwfile" "snakeoilpw");
            in
            {
              requiredBy = [ "prometheus-nextcloud-exporter.service" ];
              before = [ "prometheus-nextcloud-exporter.service" ];
              serviceConfig.ExecStart = ''
                ${pkgs.coreutils}/bin/install -o nextcloud-exporter -m 0400 ${passfile} /var/nextcloud-pwfile
              '';
            };
          services.nginx = {
            enable = true;
            virtualHosts."localhost" = {
              basicAuth.nextcloud-exporter = "snakeoilpw";
              locations."/" = {
                root = "${pkgs.prometheus-nextcloud-exporter.src}/serverinfo/testdata";
                tryFiles = "/negative-space.json =404";
              };
            };
          };
        };
        exporterTest = ''
          wait_for_unit("nginx.service")
          wait_for_unit("prometheus-nextcloud-exporter.service")
          wait_for_open_port(9205)
          succeed("curl -sSf http://localhost:9205/metrics | grep 'nextcloud_up 1'")
        '';
      };

    nginx =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          constLabels = [ "foo=bar" ];
        };
        metricProvider = {
          services.nginx = {
            enable = true;
            statusPage = true;
            virtualHosts."test".extraConfig = "return 204;";
          };
        };
        exporterTest = ''
          wait_for_unit("nginx.service")
          wait_for_unit("prometheus-nginx-exporter.service")
          wait_for_open_port(9113)
          succeed("curl -sSf http://localhost:9113/metrics | grep 'nginx_up{foo=\"bar\"} 1'")
        '';
      };

    nginxlog =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          group = "nginx";
          settings = {
            namespaces = [
              {
                name = "filelogger";
                source = {
                  files = [ "/var/log/nginx/filelogger.access.log" ];
                };
              }
              {
                name = "syslogger";
                source = {
                  syslog = {
                    listen_address = "udp://127.0.0.1:10000";
                    format = "rfc3164";
                    tags = [ "nginx" ];
                  };
                };
              }
            ];
          };
        };
        metricProvider = {
          services.nginx = {
            enable = true;
            httpConfig = ''
              server {
                listen 80;
                server_name filelogger.local;
                access_log /var/log/nginx/filelogger.access.log;
              }
              server {
                listen 81;
                server_name syslogger.local;
                access_log syslog:server=127.0.0.1:10000,tag=nginx,severity=info;
              }
            '';
          };
        };
        exporterTest = ''
          wait_for_unit("nginx.service")
          wait_for_unit("prometheus-nginxlog-exporter.service")
          wait_for_open_port(9117)
          wait_for_open_port(80)
          wait_for_open_port(81)
          succeed("curl http://localhost")
          execute("sleep 1")
          succeed(
              "curl -sSf http://localhost:9117/metrics | grep 'filelogger_http_response_count_total' | grep 1"
          )
          succeed("curl http://localhost:81")
          execute("sleep 1")
          succeed(
              "curl -sSf http://localhost:9117/metrics | grep 'syslogger_http_response_count_total' | grep 1"
          )
        '';
      };

    node =
      { ... }:
      {
        exporterConfig = {
          enable = true;
        };
        exporterTest = ''
          wait_for_unit("prometheus-node-exporter.service")
          wait_for_open_port(9100)
          succeed(
              "curl -sSf http://localhost:9100/metrics | grep 'node_exporter_build_info{.\\+} 1'"
          )
        '';
      };

    node-cert =
      { pkgs, ... }:
      {
        nodeName = "node_cert";
        exporterConfig = {
          enable = true;
          paths = [ "/run/certs" ];
        };
        exporterTest = ''
          wait_for_unit("prometheus-node-cert-exporter.service")
          wait_for_open_port(9141)
          succeed("test -f /run/certs/node-cert.cert")
          wait_until_succeeds(
              "curl -sSf http://localhost:9141/metrics | grep 'ssl_certificate_expiry_seconds{.\\+path=\"/run/certs/node-cert\\.cert\".\\+}'"
          )
        '';

        metricProvider =
          { config, ... }:
          {
            systemd.services.prometheus-node-cert-exporter = {
              serviceConfig.ExecStartPre =
                let
                  createDir = pkgs.writeShellApplication {
                    name = "create-dir";
                    text =
                      let
                        inherit (config.services.prometheus.exporters.node-cert) user;
                      in
                      ''
                        mkdir -p /run/certs
                        chown ${user}:${user} /run/certs
                        chmod ug+rwx /run/certs
                      '';
                  };
                  createCerts = pkgs.writeShellApplication {
                    name = "create-certs";
                    text = ''
                      cd /run/certs

                      cat >ca.template <<EOF
                      organization = "prometheus-node-cert-exporter"
                      cn = "prometheus-node-cert-exporter"
                      expiration_days = 365
                      ca
                      cert_signing_key
                      crl_signing_key
                      EOF

                      ${pkgs.gnutls}/bin/certtool  \
                        --generate-privkey         \
                        --key-type rsa             \
                        --sec-param High           \
                        --outfile node-cert.key

                      ${pkgs.gnutls}/bin/certtool     \
                        --generate-self-signed        \
                        --load-privkey node-cert.key  \
                        --template ca.template        \
                        --outfile node-cert.cert
                    '';
                  };
                in
                [
                  "+${lib.getExe createDir}"
                  "${lib.getExe createCerts}"
                ];
            };
          };
      };

    pgbouncer =
      { pkgs, lib, ... }:
      {
        exporterConfig = {
          enable = true;
          connectionEnvFile = "${pkgs.writeText "connstr-env" ''
            PGBOUNCER_EXPORTER_CONNECTION_STRING=postgres://admin@localhost:6432/pgbouncer?sslmode=disable
          ''}";
        };

        metricProvider = {
          services.postgresql.enable = true;
          services.pgbouncer = {
            enable = true;
            settings = {
              pgbouncer = {
                listen_addr = "*";
                auth_type = "any";
                max_client_conn = 99;
                # https://github.com/prometheus-community/pgbouncer_exporter#pgbouncer-configuration
                ignore_startup_parameters = "extra_float_digits";
              };
              databases = {
                postgres = lib.concatStringsSep " " [
                  "host=/run/postgresql"
                  "port=5432"
                  "auth_user=postgres"
                  "dbname=postgres"
                ];
              };
            };
          };
        };
        exporterTest = ''
          wait_for_unit("postgresql.target")
          wait_for_unit("pgbouncer.service")
          wait_for_unit("prometheus-pgbouncer-exporter.service")
          wait_for_open_port(9127)
          succeed("curl -sSf http://localhost:9127/metrics | grep 'pgbouncer_up 1'")
          succeed(
              "curl -sSf http://localhost:9127/metrics | grep 'pgbouncer_config_max_client_connections 99'"
          )
        '';
      };

    php-fpm =
      { pkgs, lib, ... }:
      {
        nodeName = "php_fpm";
        exporterConfig = {
          enable = true;
          environmentFile = pkgs.writeTextFile {
            name = "/tmp/prometheus-php-fpm-exporter.env";
            text = ''
              PHP_FPM_SCRAPE_URI="tcp://127.0.0.1:9000/status"
            '';
          };
        };
        metricProvider = {
          users.users."php-fpm-exporter" = {
            isSystemUser = true;
            group = "php-fpm-exporter";
          };
          users.groups."php-fpm-exporter" = { };
          services.phpfpm.pools."php-fpm-exporter" = {
            user = "php-fpm-exporter";
            group = "php-fpm-exporter";
            settings = {
              "pm" = "dynamic";
              "pm.max_children" = 32;
              "pm.max_requests" = 500;
              "pm.start_servers" = 2;
              "pm.min_spare_servers" = 2;
              "pm.max_spare_servers" = 5;
              "pm.status_path" = "/status";
              "listen" = "127.0.0.1:9000";
              "listen.allowed_clients" = "127.0.0.1";
            };
            phpEnv."PATH" = lib.makeBinPath [ pkgs.php ];
          };
        };
        exporterTest = ''
          wait_for_unit("phpfpm-php-fpm-exporter.service")
          wait_for_unit("prometheus-php-fpm-exporter.service")
          succeed("curl -sSf http://localhost:9253/metrics | grep 'phpfpm_up{.*} 1'")
        '';
      };

    ping =
      { ... }:
      {
        exporterConfig = {
          enable = true;

          settings = {
            targets = [
              {
                "localhost" = {
                  alias = "local machine";
                  env = "prod";
                  type = "domain";
                };
              }
              {
                "127.0.0.1" = {
                  alias = "local machine";
                  type = "v4";
                };
              }
              {
                "::1" = {
                  alias = "local machine";
                  type = "v6";
                };
              }
              {
                "google.com" = { };
              }
            ];
            dns = { };
            ping = {
              interval = "2s";
              timeout = "3s";
              history-size = 42;
              payload-size = 56;
            };
            log = {
              level = "warn";
            };
          };
        };

        exporterTest = ''
          wait_for_unit("prometheus-ping-exporter.service")
          wait_for_open_port(9427)
          succeed("curl -sSf http://localhost:9427/metrics | grep 'ping_up{.*} 1'")
        '';
      };

    postfix =
      { ... }:
      {
        exporterConfig = {
          enable = true;
        };
        metricProvider = {
          services.postfix.enable = true;
        };
        exporterTest = ''
          wait_for_unit("prometheus-postfix-exporter.service")
          wait_for_file("/var/lib/postfix/queue/public/showq")
          wait_for_open_port(9154)
          wait_until_succeeds(
              "curl -sSf http://localhost:9154/metrics | grep 'postfix_up{path=\"unix:///var/lib/postfix/queue/public/showq\"} 1'"
          )
          succeed(
              "curl -sSf http://localhost:9154/metrics | grep 'postfix_smtpd_connects_total 0'"
          )
          succeed("curl -sSf http://localhost:9154/metrics | grep 'postfix_up{.*} 1'")
        '';
      };

    postgres =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          runAsLocalSuperUser = true;
        };
        metricProvider = {
          services.postgresql.enable = true;
        };
        exporterTest = ''
          wait_for_unit("prometheus-postgres-exporter.service")
          wait_for_open_port(9187)
          wait_for_unit("postgresql.target")
          succeed(
              "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
          )
          succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
          systemctl("stop postgresql")
          succeed(
              "curl -sSf http://localhost:9187/metrics | grep -v 'pg_exporter_last_scrape_error 0'"
          )
          succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 0'")
          systemctl("start postgresql")
          wait_for_unit("postgresql.target")
          succeed(
              "curl -sSf http://localhost:9187/metrics | grep 'pg_exporter_last_scrape_error 0'"
          )
          succeed("curl -sSf http://localhost:9187/metrics | grep 'pg_up 1'")
        '';
      };

    process =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          settings.process_names = [
            # Remove nix store path from process name
            {
              name = "{{.Matches.Wrapped}} {{ .Matches.Args }}";
              cmdline = [ "^/nix/store[^ ]*/(?P<Wrapped>[^ /]*) (?P<Args>.*)" ];
            }
          ];
        };
        exporterTest = ''
          wait_for_unit("prometheus-process-exporter.service")
          wait_for_open_port(9256)
          wait_until_succeeds(
              "curl -sSf localhost:9256/metrics | grep -q '{}'".format(
                  'namedprocess_namegroup_cpu_seconds_total{groupname="process-exporter '
              )
          )
        '';
      };

    pve =
      { pkgs, ... }:
      let
        pveExporterEnvFile = pkgs.writeTextFile {
          name = "pve.env";
          text = ''
            PVE_USER="test_user@pam"
            PVE_PASSWORD="hunter3"
            PVE_VERIFY_SSL="false"
          '';
        };
      in
      {
        exporterConfig = {
          enable = true;
          environmentFile = pveExporterEnvFile;
        };
        exporterTest = ''
          wait_for_unit("prometheus-pve-exporter.service")
          wait_for_open_port(9221)
          wait_until_succeeds("curl localhost:9221")
        '';
      };

    py-air-control =
      { ... }:
      {
        nodeName = "py_air_control";
        exporterConfig = {
          enable = true;
          deviceHostname = "127.0.0.1";
        };
        exporterTest = ''
          wait_for_unit("prometheus-py-air-control-exporter.service")
          wait_for_open_port(9896)
          succeed(
              "curl -sSf http://localhost:9896/metrics | grep 'py_air_control_sampling_error_total'"
          )
        '';
      };

    redis =
      { ... }:
      {
        exporterConfig = {
          enable = true;
        };
        metricProvider.services.redis.servers."".enable = true;
        exporterTest = ''
          wait_for_unit("redis.service")
          wait_for_unit("prometheus-redis-exporter.service")
          wait_for_open_port(6379)
          wait_for_open_port(9121)
          wait_until_succeeds("curl -sSf localhost:9121/metrics | grep 'redis_up 1'")
        '';
      };

    restic =
      { pkgs, ... }:
      let
        repository = "rest:http://127.0.0.1:8000";
        passwordFile = pkgs.writeText "restic-test-password" "test-password";
      in
      {
        exporterConfig = {
          enable = true;
          inherit repository passwordFile;
        };
        metricProvider = {
          services.restic.server = {
            enable = true;
            extraFlags = [ "--no-auth" ];
          };
          environment.systemPackages = [ pkgs.restic ];
        };
        exporterTest = ''
          # prometheus-restic-exporter.service fails without initialised repository
          systemctl("stop prometheus-restic-exporter.service")

          # Initialise the repository
          wait_for_unit("restic-rest-server.service")
          wait_for_open_port(8000)
          succeed("restic init --repo ${repository} --password-file ${passwordFile}")

          systemctl("start prometheus-restic-exporter.service")
          wait_for_unit("prometheus-restic-exporter.service")
          wait_for_open_port(9753)
          wait_until_succeeds("curl -sSf localhost:9753/metrics | grep 'restic_check_success 1.0'")
        '';
      };

    rtl_433 =
      { ... }:
      {
        pkgsReadOnly = false;
        exporterConfig = {
          enable = true;
        };
        metricProvider = {
          # Mock rtl_433 binary to return a dummy metric stream.
          nixpkgs.overlays = [
            (self: super: {
              rtl_433 = self.runCommand "rtl_433" { } ''
                mkdir -p "$out/bin"
                cat <<EOF > "$out/bin/rtl_433"
                #!/bin/sh
                while true; do
                  printf '{"time" : "2020-04-26 13:37:42", "model" : "zopieux", "id" : 55, "channel" : 3, "temperature_C" : 18.000}\n'
                  sleep 4
                done
                EOF
                chmod +x "$out/bin/rtl_433"
              '';
            })
          ];
        };
        exporterTest = ''
          wait_for_unit("prometheus-rtl_433-exporter.service")
          wait_for_open_port(9550)
          wait_until_succeeds(
              "curl -sSf localhost:9550/metrics | grep '{}'".format(
                  'rtl_433_temperature_celsius{channel="3",id="55",location="",model="zopieux"} 18'
              )
          )
        '';
      };

    sabnzbd =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          servers = [
            {
              baseUrl = "http://localhost:8080";
              apiKeyFile = "/var/sabnzbd-apikey";
            }
          ];
        };

        metricProvider = {
          services.sabnzbd.enable = true;

          # extract the generated api key before starting
          systemd.services.sabnzbd-apikey = {
            requires = [ "sabnzbd.service" ];
            after = [ "sabnzbd.service" ];
            requiredBy = [ "prometheus-sabnzbd-exporter.service" ];
            before = [ "prometheus-sabnzbd-exporter.service" ];
            script = ''
              grep -Po '^api_key = \K.+' /var/lib/sabnzbd/sabnzbd.ini > /var/sabnzbd-apikey
            '';
          };
        };

        exporterTest = ''
          wait_for_unit("sabnzbd.service")
          wait_for_unit("prometheus-sabnzbd-exporter.service")
          wait_for_open_port(8080)
          wait_for_open_port(9387)
          wait_until_succeeds(
              "curl -sSf 'localhost:9387/metrics' | grep 'sabnzbd_queue_size{sabnzbd_instance=\"http://localhost:8080\"} 0.0'"
          )
        '';
      };

    scaphandre =
      { ... }:
      {
        exporterConfig = {
          enable = true;
        };
        metricProvider = {
          boot.kernelModules = [ "intel_rapl_common" ];
        };
        exporterTest = ''
          wait_for_unit("prometheus-scaphandre-exporter.service")
          wait_for_open_port(8080)
          wait_until_succeeds(
              "curl -sSf 'localhost:8080/metrics'"
          )
        '';
      };

    shelly =
      { pkgs, ... }:
      {
        exporterConfig = {
          enable = true;
          metrics-file = "${pkgs.writeText "test.json" "{}"}";
        };
        exporterTest = ''
          wait_for_unit("prometheus-shelly-exporter.service")
          wait_for_open_port(9784)
          wait_until_succeeds(
              "curl -sSf 'localhost:9784/metrics'"
          )
        '';
      };

    script =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          settings.scripts = [
            {
              name = "success";
              command = [ "sleep" ];
              args = [ "1" ];
            }
          ];
        };
        exporterTest = ''
          wait_for_unit("prometheus-script-exporter.service")
          wait_for_open_port(9172)
          wait_until_succeeds(
              "curl -sSf 'localhost:9172/probe?script=success' | grep -q '{}'".format(
                  'script_success{script="success"} 1'
              )
          )
        '';
      };

    smartctl =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          devices = [
            "/dev/vda"
          ];
        };
        exporterTest = ''
          wait_until_succeeds(
              'journalctl -eu prometheus-smartctl-exporter.service -o cat | grep "Unable to detect device type"'
          )
        '';
      };

    smokeping =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          hosts = [ "127.0.0.1" ];
        };
        exporterTest = ''
          wait_for_unit("prometheus-smokeping-exporter.service")
          wait_for_open_port(9374)
          wait_until_succeeds(
              "curl -sSf localhost:9374/metrics | grep '{}' | grep -v ' 0$'".format(
                  'smokeping_requests_total{host="127.0.0.1",ip="127.0.0.1",source="",tos="0"} '
              )
          )
          wait_until_succeeds(
              "curl -sSf localhost:9374/metrics | grep '{}'".format(
                  'smokeping_response_ttl{host="127.0.0.1",ip="127.0.0.1",source="",tos="0"}'
              )
          )
        '';
      };

    storagebox =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          tokenFile = "/tmp/faketoken";
        };
        metricProvider = {
          networking = {
            # The exporter tries to access Hetzner on startup and crashes.
            # Blocking this on the firewall level allows the exporter to start.
            extraHosts = "127.0.0.1 api.hetzner.com";
            firewall.extraCommands = ''
              iptables -A OUTPUT -p tcp --dport 443 -d 127.0.0.1 -j DROP
            '';
          };
        };
        exporterTest = ''
          succeed(
            'echo faketoken > /tmp/faketoken'
          )
          wait_for_unit("prometheus-storagebox-exporter.service")
          wait_for_open_port(9509)
          succeed("curl -sSf localhost:9509/metrics | grep 'process_open_fds'")
        '';
      };

    snmp =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          configuration = {
            auths.public_v2 = {
              community = "public";
              version = 2;
            };
            modules.test = { };
          };
        };
        exporterTest = ''
          wait_for_unit("prometheus-snmp-exporter.service")
          wait_for_open_port(9116)
          succeed("curl -sSf localhost:9116/metrics | grep 'snmp_request_errors_total 0'")
        '';
      };

    sql =
      { ... }:
      {
        exporterConfig = {
          configuration.jobs.points = {
            interval = "1m";
            connections = [
              "postgres://prometheus-sql-exporter@/data?host=/run/postgresql&sslmode=disable"
            ];
            queries = {
              points = {
                labels = [ "name" ];
                help = "Amount of points accumulated per person";
                values = [ "amount" ];
                query = "SELECT SUM(amount) as amount, name FROM points GROUP BY name";
              };
            };
          };
          enable = true;
          user = "prometheus-sql-exporter";
        };
        metricProvider = {
          services.postgresql = {
            enable = true;
            initialScript = builtins.toFile "init.sql" ''
              CREATE DATABASE data;
              \c data;
              CREATE TABLE points (amount INT, name TEXT);
              INSERT INTO points(amount, name) VALUES (1, 'jack');
              INSERT INTO points(amount, name) VALUES (2, 'jill');
              INSERT INTO points(amount, name) VALUES (3, 'jack');

              CREATE USER "prometheus-sql-exporter";
              GRANT ALL PRIVILEGES ON DATABASE data TO "prometheus-sql-exporter";
              GRANT SELECT ON points TO "prometheus-sql-exporter";
            '';
          };
          systemd.services.prometheus-sql-exporter.after = [ "postgresql.target" ];
        };
        exporterTest = ''
          wait_for_unit("prometheus-sql-exporter.service")
          wait_for_open_port(9237)
          succeed("curl http://localhost:9237/metrics | grep -c 'sql_points{' | grep 2")
        '';
      };

    statsd =
      { ... }:
      {
        exporterConfig = {
          enable = true;
        };
        exporterTest = ''
          wait_for_unit("prometheus-statsd-exporter.service")
          wait_for_open_port(9102)
          succeed("curl http://localhost:9102/metrics | grep 'statsd_exporter_build_info{'")
          wait_until_succeeds(
            "echo 'test.udp:1|c' > /dev/udp/localhost/9125 && \
            curl http://localhost:9102/metrics | grep 'test_udp 1'",
            timeout=10
          )
          wait_until_succeeds(
            "echo 'test.tcp:1|c' > /dev/tcp/localhost/9125 && \
            curl http://localhost:9102/metrics | grep 'test_tcp 1'",
            timeout=10
          )
        '';
      };

    surfboard =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          modemAddress = "localhost";
        };
        metricProvider = {
          systemd.services.prometheus-surfboard-exporter.after = [ "nginx.service" ];
          services.nginx = {
            enable = true;
            virtualHosts.localhost.locations."/cgi-bin/status".extraConfig = ''
              return 204;
            '';
          };
        };
        exporterTest = ''
          wait_for_unit("nginx.service")
          wait_for_open_port(80)
          wait_for_unit("prometheus-surfboard-exporter.service")
          wait_for_open_port(9239)
          succeed("curl -sSf localhost:9239/metrics | grep 'surfboard_up 1'")
        '';
      };

    systemd =
      { ... }:
      {
        exporterConfig = {
          enable = true;

          extraFlags = [
            "--systemd.collector.enable-restart-count"
          ];
        };
        metricProvider = { };
        exporterTest = ''
          wait_for_unit("prometheus-systemd-exporter.service")
          wait_for_open_port(9558)
          wait_until_succeeds(
              "curl -sSf localhost:9558/metrics | grep '{}'".format(
                  'systemd_unit_state{name="basic.target",state="active",type="target"} 1'
              )
          )
          succeed(
              "curl -sSf localhost:9558/metrics | grep '{}'".format(
                  'systemd_service_restart_total{name="prometheus-systemd-exporter.service"} 0'
              )
          )
        '';
      };

    tailscale =
      { pkgs, ... }:
      {
        exporterConfig = {
          package = pkgs.prometheus-tailscale-exporter.overrideAttrs {
            patches = [
              # This patch prevents the exporter from exiting immediately upon
              # startup when no credentials are provided, which is useful for
              # testing the NixOS module.
              (pkgs.writeText "allow-running-without-credentials" ''
                diff --git a/cmd/tailscale-exporter/root.go b/cmd/tailscale-exporter/root.go
                index 14089f9..2bb9a25 100644
                --- a/cmd/tailscale-exporter/root.go
                +++ b/cmd/tailscale-exporter/root.go
                @@ -162,13 +162,6 @@ func runExporter(cmd *cobra.Command, args []string) error {
                ''\t''\t}

                ''\t''\thttpClient := oauthConfig.Client(context.Background())
                -''\t''\ttoken, err := oauthConfig.Token(context.Background())
                -''\t''\tif err != nil {
                -''\t''\t''\treturn fmt.Errorf("failed to obtain OAuth token: %w", err)
                -''\t''\t}
                -''\t''\tlogger.Info("OAuth token obtained", "token_type", token.TokenType)
                -''\t''\tlogger.Info("Successfully obtained OAuth token", "expires", token.Expiry)
                -
                ''\t''\ttsCollector, err := tailscale.NewTailscaleCollector(
                ''\t''\t''\tlogger,
                ''\t''\t''\thttpClient,
              '')
            ];
          };
          enable = true;
          environmentFile = pkgs.writeText "tailscale-exporter-env" ''
            TAILSCALE_OAUTH_CLIENT_ID=12345678
            TAILSCALE_OAUTH_CLIENT_SECRET=12345678
            TAILSCALE_TAILNET=example.com
          '';
        };

        exporterTest = ''
          wait_for_unit("prometheus-tailscale-exporter.service")
          wait_for_open_port(9250)
          succeed("curl -sSf localhost:9250/metrics | grep 'tailscale_up{tailnet=\"example.com\"} 1'")
        '';
      };

    unpoller =
      { ... }:
      {
        nodeName = "unpoller";
        exporterConfig.enable = true;
        exporterConfig.controllers = [ { } ];
        exporterTest = ''
          wait_until_succeeds(
              'journalctl -eu prometheus-unpoller-exporter.service -o cat | grep "Connection Error"'
          )
        '';
      };

    unbound =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          unbound.host = "unix:///run/unbound/unbound.ctl";
        };
        metricProvider = {
          services.unbound = {
            enable = true;
            localControlSocketPath = "/run/unbound/unbound.ctl";
          };
          systemd.services.prometheus-unbound-exporter.serviceConfig = {
            SupplementaryGroups = [ "unbound" ];
          };
        };
        exporterTest = ''
          wait_for_unit("unbound.service")
          wait_for_unit("prometheus-unbound-exporter.service")
          wait_for_open_port(9167)
          wait_until_succeeds("curl -sSf localhost:9167/metrics | grep 'unbound_up 1'")
        '';
      };

    v2ray =
      { ... }:
      {
        exporterConfig = {
          enable = true;
        };

        metricProvider = {
          systemd.services.prometheus-nginx-exporter.after = [ "v2ray.service" ];
          services.v2ray = {
            enable = true;
            config = {
              stats = { };
              api = {
                tag = "api";
                services = [ "StatsService" ];
              };
              inbounds = [
                {
                  port = 1080;
                  listen = "127.0.0.1";
                  protocol = "http";
                }
                {
                  listen = "127.0.0.1";
                  port = 54321;
                  protocol = "dokodemo-door";
                  settings = {
                    address = "127.0.0.1";
                  };
                  tag = "api";
                }
              ];
              outbounds = [
                {
                  protocol = "freedom";
                }
                {
                  protocol = "freedom";
                  settings = { };
                  tag = "api";
                }
              ];
              routing = {
                strategy = "rules";
                settings = {
                  rules = [
                    {
                      inboundTag = [ "api" ];
                      outboundTag = "api";
                      type = "field";
                    }
                  ];
                };
              };
            };
          };
        };
        exporterTest = ''
          wait_for_unit("prometheus-v2ray-exporter.service")
          wait_for_open_port(9299)
          succeed("curl -sSf localhost:9299/scrape | grep 'v2ray_up 1'")
        '';
      };

    varnish =
      { ... }:
      {
        exporterConfig = {
          enable = true;
          instance = "/var/run/varnishd";
          group = "varnish";
        };
        metricProvider = {
          systemd.services.prometheus-varnish-exporter.after = [
            "varnish.service"
          ];
          services.varnish = {
            enable = true;
            config = ''
              vcl 4.0;
              backend default {
                .host = "127.0.0.1";
                .port = "80";
              }
            '';
          };
        };
        exporterTest = ''
          wait_for_unit("prometheus-varnish-exporter.service")
          wait_for_open_port(6081)
          wait_for_open_port(9131)
          succeed("curl -sSf http://localhost:9131/metrics | grep 'varnish_up 1'")
        '';
      };

    wireguard =
      { pkgs, lib, ... }:
      let
        snakeoil = import ./wireguard/snakeoil-keys.nix;
        publicKeyWithoutNewlines = lib.replaceStrings [ "\n" ] [ "" ] snakeoil.peer1.publicKey;
      in
      {
        exporterConfig.enable = true;
        metricProvider = {
          networking.wireguard.interfaces.wg0 = {
            ips = [
              "10.23.42.1/32"
              "fc00::1/128"
            ];
            listenPort = 23542;

            inherit (snakeoil.peer0) privateKey;

            peers = lib.singleton {
              allowedIPs = [
                "10.23.42.2/32"
                "fc00::2/128"
              ];

              inherit (snakeoil.peer1) publicKey;
            };
          };
          systemd.services.prometheus-wireguard-exporter.after = [ "wireguard-wg0.service" ];
        };
        exporterTest = ''
          wait_for_unit("prometheus-wireguard-exporter.service")
          wait_for_open_port(9586)
          wait_until_succeeds(
              "curl -sSf http://localhost:9586/metrics | grep '${publicKeyWithoutNewlines}'"
          )
        '';
      };

    zfs =
      { ... }:
      {
        exporterConfig = {
          enable = true;
        };
        metricProvider = {
          boot.supportedFilesystems = [ "zfs" ];
          networking.hostId = "7327ded7";
        };
        exporterTest = ''
          wait_for_unit("prometheus-zfs-exporter.service")
          wait_for_unit("zfs.target")
          wait_for_open_port(9134)
          wait_until_succeeds("curl -f localhost:9134/metrics | grep 'zfs_scrape_collector_success{.*} 1'")
        '';
      };
  };
in
lib.mapAttrs (
  exporter: testConfigFun:
  (runTest (
    { pkgs, lib, ... }:
    let
      testConfig = testConfigFun { inherit pkgs lib; };
      nodeName = testConfig.nodeName or exporter;
    in
    {
      name = "prometheus-${exporter}-exporter";
      node.pkgsReadOnly = testConfig.pkgsReadOnly or true;

      nodes.${nodeName} = lib.mkMerge [
        {
          services.prometheus.exporters.${exporter} = testConfig.exporterConfig;
        }
        testConfig.metricProvider or { }
      ];

      testScript = ''
        ${nodeName}.start()
        ${lib.concatStringsSep "\n" (
          map (
            line:
            if
              builtins.any (b: b) [
                (builtins.match "^[[:space:]]*$" line != null)
                (builtins.substring 0 1 line == "#")
                (builtins.substring 0 1 line == " ")
                (builtins.substring 0 1 line == ")")
              ]
            then
              line
            else
              "${nodeName}.${line}"
          ) (lib.splitString "\n" (lib.removeSuffix "\n" testConfig.exporterTest))
        )}
        ${nodeName}.shutdown()
      '';

      meta.maintainers = [ ];
    }
  ))
) exporterTests
