c4patino/yumeami
1
0
Fork 0
Code Activity Actions

fix: add mnt-nfs-slurm.mount dependency to slurmctl processes

Browse source
This commit is contained in:
Ceferino Patino 2026-06-27 14:51:48 -05:00
commit 83e97c1c8d
Signed by: c4patino
SSH key fingerprint: SHA256:Wu+cU1t+7zVT9wzew/4meNmeulo66NzMqc22MdDBgXI
1 changed files with 79 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

142
modules/nixos/services/ci/slurm/default.nix
View file

@ -5,7 +5,7 @@
namespace, namespace,
... ...
}: let }: let
inherit (lib) types mkEnableOption mkOption mkIf groupBy mapAttrsToList attrNames mapAttrs concatStringsSep flatten; inherit (lib) types mkEnableOption mkIf groupBy mapAttrsToList attrNames mapAttrs concatStringsSep flatten;
inherit (lib.${namespace}) getAttrByNamespace mkOptionsWithNamespace resolveHostIP mkOpt mkOptAttrset mkListOpt; inherit (lib.${namespace}) getAttrByNamespace mkOptionsWithNamespace resolveHostIP mkOpt mkOptAttrset mkListOpt;
inherit (config.networking) hostName; inherit (config.networking) hostName;
base = "${namespace}.services.ci.slurm"; base = "${namespace}.services.ci.slurm";
@ -25,78 +25,91 @@ in {
}; };
config = mkIf cfg.enable { config = mkIf cfg.enable {
services.slurm = { services = {
client.enable = builtins.hasAttr hostName cfg.nodeMap; slurm = {
server.enable = builtins.elem hostName cfg.controlHosts; client.enable = builtins.hasAttr hostName cfg.nodeMap;
server.enable = builtins.elem hostName cfg.controlHosts;
stateSaveLocation = "/mnt/nfs/slurm"; stateSaveLocation = "/mnt/nfs/slurm";
nodeName = let nodeName = let
mkNodeConfig = node: info: "${node} NodeAddr=${resolveHostIP networkCfg.devices node} ${info.configString} State=UNKNOWN"; mkNodeConfig = node: info: "${node} NodeAddr=${resolveHostIP networkCfg.devices node} ${info.configString} State=UNKNOWN";
in in
mapAttrsToList mkNodeConfig cfg.nodeMap; mapAttrsToList mkNodeConfig cfg.nodeMap;
partitionName = let partitionName = let
mkPartitions = nodeMap: mkPartitions = nodeMap:
nodeMap nodeMap
|> attrNames |> attrNames
|> map ( |> map (
node: node:
(cfg.nodeMap.${node}.partitions) (cfg.nodeMap.${node}.partitions)
|> map (partition: {inherit partition node;}) |> map (partition: {inherit partition node;})
) )
|> flatten |> flatten
|> groupBy (x: x.partition) |> groupBy (x: x.partition)
|> mapAttrs (name: entries: entries |> map (e: e.node)); |> mapAttrs (name: entries: entries |> map (e: e.node));
formatPartition = name: nodes: "${name} Nodes=${nodes |> concatStringsSep ","} Default=${ formatPartition = name: nodes: "${name} Nodes=${nodes |> concatStringsSep ","} Default=${
if name == "main" if name == "main"
then "YES" then "YES"
else "NO" else "NO"
} MaxTime=INFINITE State=UP"; } MaxTime=INFINITE State=UP";
in in
cfg.nodeMap cfg.nodeMap
|> mkPartitions |> mkPartitions
|> mapAttrsToList formatPartition; |> mapAttrsToList formatPartition;
extraConfig = let extraConfig = let
mkHostString = host: "SlurmctldHost=${host}(${resolveHostIP networkCfg.devices host})"; mkHostString = host: "SlurmctldHost=${host}(${resolveHostIP networkCfg.devices host})";
hostStrings = hostStrings =
cfg.controlHosts cfg.controlHosts
|> map mkHostString |> map mkHostString
|> concatStringsSep "\n"; |> concatStringsSep "\n";
in '' in ''
${hostStrings} ${hostStrings}
GresTypes=gpu,shard GresTypes=gpu,shard
TaskPlugin=task/cgroup TaskPlugin=task/cgroup
SlurmdParameters=allow_ecores SlurmdParameters=allow_ecores
DefCpuPerGPU=1 DefCpuPerGPU=1
DefMemPerCPU=1000 DefMemPerCPU=1000
ReturnToService=2 ReturnToService=2
TaskProlog=${inputs.dotfiles + "/slurm/prolog.sh"} TaskProlog=${inputs.dotfiles + "/slurm/prolog.sh"}
TaskEpilog=${inputs.dotfiles + "/slurm/epilog.sh"} TaskEpilog=${inputs.dotfiles + "/slurm/epilog.sh"}
''; '';
extraCgroupConfig = extraCgroupConfig =
''
ConstrainCores=yes
ConstrainDevices=yes
ConstrainRAMSpace=yes
''
+ (
if hostName != "chibi"
then ''
ConstrainSwapSpace=yes
AllowedSwapSpace=0
'' ''
else '''' ConstrainCores=yes
); ConstrainDevices=yes
ConstrainRAMSpace=yes
''
+ (
if hostName != "chibi"
then ''
ConstrainSwapSpace=yes
AllowedSwapSpace=0
''
else ''''
);
extraConfigPaths = [(inputs.dotfiles + "/slurm/config")]; extraConfigPaths = [(inputs.dotfiles + "/slurm/config")];
};
munge.enable = true;
};
systemd.services.slurmctld = mkIf (builtins.elem hostName cfg.controlHosts) {
requires = [
"mnt-nfs-slurm.mount"
];
after = [
"mnt-nfs-slurm.mount"
];
}; };
services.munge.enable = true;
environment.etc."munge/munge.key" = { environment.etc."munge/munge.key" = {
source = "${inputs.self}/secrets/crypt/munge.key"; source = "${inputs.self}/secrets/crypt/munge.key";
user = "munge"; user = "munge";
@ -104,6 +117,9 @@ in {
mode = "0400"; mode = "0400";
}; };
${namespace}.services.storage.impermanence.folders = ["/var/spool/slurmctld" "/var/spool/slurmd"]; ${namespace}.services.storage.impermanence.folders = [
"/var/spool/slurmctld"
"/var/spool/slurmd"
];
}; };
} }