diff --git a/modules/nixos/services/apps/slurm/client.nix b/modules/nixos/services/apps/slurm/client.nix new file mode 100644 index 0000000..eeca17b --- /dev/null +++ b/modules/nixos/services/apps/slurm/client.nix @@ -0,0 +1,40 @@ +{ + config, + inputs, + lib, + namespace, + ... +}: let + inherit (lib) mkIf mkAfter optionalString; + inherit (lib.${namespace}) getAttrByNamespace mkPersistDir; + base = "${namespace}.services.apps.slurm"; + cfg = getAttrByNamespace config base; + inherit (config.networking) hostName; +in { + config = mkIf (builtins.hasAttr hostName cfg.nodeMap) { + services.slurm = { + client.enable = true; + + extraConfig = mkAfter '' + SlurmdParameters=allow_ecores + + TaskProlog=${inputs.dotfiles + "/slurm/prolog.sh"} + TaskEpilog=${inputs.dotfiles + "/slurm/epilog.sh"} + ''; + + extraCgroupConfig = '' + ConstrainCores=yes + ConstrainDevices=yes + ConstrainRAMSpace=yes + '' + + optionalString (hostName != "chibi") '' + ConstrainSwapSpace=yes + AllowedSwapSpace=0 + ''; + }; + + ${namespace}.services.storage.impermanence.folders = [ + (mkPersistDir config "slurm" "/var/spool/slurmd") + ]; + }; +} diff --git a/modules/nixos/services/apps/slurm/default.nix b/modules/nixos/services/apps/slurm/default.nix index 7e3d07e..1167f4c 100644 --- a/modules/nixos/services/apps/slurm/default.nix +++ b/modules/nixos/services/apps/slurm/default.nix @@ -7,11 +7,15 @@ }: let inherit (lib) types mkEnableOption mkIf groupBy mapAttrsToList attrNames mapAttrs concatStringsSep flatten; inherit (lib.${namespace}) getAttrByNamespace mkOptionsWithNamespace resolveHostIP mkOpt mkOptAttrset mkListOpt mkPersistDir; - inherit (config.networking) hostName; base = "${namespace}.services.apps.slurm"; cfg = getAttrByNamespace config base; networkCfg = getAttrByNamespace config "${namespace}.services.networking"; in { + imports = [ + ./client.nix + ./server.nix + ]; + options = with types; mkOptionsWithNamespace base { enable = mkEnableOption "SLURM"; @@ -27,11 +31,6 @@ in { config = mkIf cfg.enable { services = { slurm = { - client.enable = builtins.hasAttr hostName cfg.nodeMap; - server.enable = builtins.elem hostName cfg.controlHosts; - - stateSaveLocation = "/mnt/nfs/slurm"; - nodeName = let mkNodeConfig = node: info: "${node} NodeAddr=${resolveHostIP networkCfg.devices node} ${info.configString} State=UNKNOWN"; in @@ -61,39 +60,19 @@ in { |> mapAttrsToList formatPartition; extraConfig = let - mkHostString = host: "SlurmctldHost=${host}(${resolveHostIP networkCfg.devices host})"; hostStrings = cfg.controlHosts - |> map mkHostString + |> map (host: "SlurmctldHost=${host}(${resolveHostIP networkCfg.devices host})") |> concatStringsSep "\n"; in '' ${hostStrings} GresTypes=gpu,shard TaskPlugin=task/cgroup - SlurmdParameters=allow_ecores DefCpuPerGPU=1 DefMemPerCPU=1000 ReturnToService=2 - - TaskProlog=${inputs.dotfiles + "/slurm/prolog.sh"} - TaskEpilog=${inputs.dotfiles + "/slurm/epilog.sh"} ''; - extraCgroupConfig = - '' - ConstrainCores=yes - ConstrainDevices=yes - ConstrainRAMSpace=yes - '' - + ( - if hostName != "chibi" - then '' - ConstrainSwapSpace=yes - AllowedSwapSpace=0 - '' - else '''' - ); - extraConfigPaths = [(inputs.dotfiles + "/slurm/config")]; }; @@ -103,16 +82,6 @@ in { }; }; - systemd.services.slurmctld = mkIf (builtins.elem hostName cfg.controlHosts) { - requires = [ - "mnt-nfs-slurm.mount" - ]; - - after = [ - "mnt-nfs-slurm.mount" - ]; - }; - sops.secrets = let inherit (config.users.users) munge; in { @@ -126,8 +95,6 @@ in { }; ${namespace}.services.storage.impermanence.folders = [ - (mkPersistDir config "slurm" "/var/spool/slurmctld") - (mkPersistDir config "slurm" "/var/spool/slurmd") (mkPersistDir config "munge" "/var/lib/munge") ]; }; diff --git a/modules/nixos/services/apps/slurm/server.nix b/modules/nixos/services/apps/slurm/server.nix new file mode 100644 index 0000000..9a7a658 --- /dev/null +++ b/modules/nixos/services/apps/slurm/server.nix @@ -0,0 +1,28 @@ +{ + config, + lib, + namespace, + ... +}: let + inherit (lib) mkIf; + inherit (lib.${namespace}) getAttrByNamespace mkPersistDir; + base = "${namespace}.services.apps.slurm"; + cfg = getAttrByNamespace config base; + inherit (config.networking) hostName; +in { + config = mkIf (builtins.elem hostName cfg.controlHosts) { + services.slurm = { + server.enable = true; + stateSaveLocation = "/mnt/nfs/slurm"; + }; + + systemd.services.slurmctld = { + requires = ["mnt-nfs-slurm.mount"]; + after = ["mnt-nfs-slurm.mount"]; + }; + + ${namespace}.services.storage.impermanence.folders = [ + (mkPersistDir config "slurm" "/var/spool/slurmctld") + ]; + }; +}