mirror of
https://github.com/NixOS/nixpkgs.git
synced 2026-07-06 17:13:24 -05:00
nixos/llama-cpp: support models-preset
The new options allows specifying --models-preset via Nix.
This commit is contained in:
parent
97b35f6420
commit
676d8fc1bc
1 changed files with 36 additions and 0 deletions
|
|
@ -8,6 +8,12 @@
|
|||
|
||||
let
|
||||
cfg = config.services.llama-cpp;
|
||||
|
||||
modelsPresetFile =
|
||||
if cfg.modelsPreset != null then
|
||||
pkgs.writeText "llama-models.ini" (lib.generators.toINI { } cfg.modelsPreset)
|
||||
else
|
||||
null;
|
||||
in
|
||||
{
|
||||
|
||||
|
|
@ -32,6 +38,32 @@ in
|
|||
default = null;
|
||||
};
|
||||
|
||||
modelsPreset = lib.mkOption {
|
||||
type = lib.types.nullOr (lib.types.attrsOf lib.types.attrs);
|
||||
default = null;
|
||||
description = ''
|
||||
Models preset configuration as a Nix attribute set.
|
||||
This is converted to an INI file and passed to llama-server via --model-preset.
|
||||
See llama-server documentation for available options.
|
||||
'';
|
||||
example = lib.literalExpression ''
|
||||
{
|
||||
"Qwen3-Coder-Next" = {
|
||||
hf-repo = "unsloth/Qwen3-Coder-Next-GGUF";
|
||||
hf-file = "Qwen3-Coder-Next-UD-Q4_K_XL.gguf";
|
||||
alias = "unsloth/Qwen3-Coder-Next";
|
||||
fit = "on";
|
||||
seed = "3407";
|
||||
temp = "1.0";
|
||||
top-p = "0.95";
|
||||
min-p = "0.01";
|
||||
top-k = "40";
|
||||
jinja = "on";
|
||||
};
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
extraFlags = lib.mkOption {
|
||||
type = lib.types.listOf lib.types.str;
|
||||
description = "Extra flags passed to llama-cpp-server.";
|
||||
|
|
@ -97,6 +129,10 @@ in
|
|||
"--models-dir"
|
||||
cfg.modelsDir
|
||||
]
|
||||
++ lib.optionals (cfg.modelsPreset != null) [
|
||||
"--models-preset"
|
||||
modelsPresetFile
|
||||
]
|
||||
++ cfg.extraFlags;
|
||||
in
|
||||
"${cfg.package}/bin/llama-server ${utils.escapeSystemdExecArgs args}";
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue