diff --git a/ansible/host_vars/ci-01.lab.alexpires.me.yml b/ansible/host_vars/ci-01.lab.alexpires.me.yml index 8d2cad6..1cfcb39 100644 --- a/ansible/host_vars/ci-01.lab.alexpires.me.yml +++ b/ansible/host_vars/ci-01.lab.alexpires.me.yml @@ -29,3 +29,58 @@ windows_exporter_collectors: - tcp - time - update + +llama_server_windows_install_dir: "D:\\llama_server" +llama_server_windows_user: "llama" +llama_server_windows_working_dir: "D:\\llama_server" +llama_server_cuda_version: "13.3" + +llama_server_models: + - name: "luau-qwen3-4b-fim-v0.1" + model_id: "mradermacher/Luau-Qwen3-4B-FIM-v0.1-GGUF" + quant: "IQ4_XS" + revision: "main" + - name: "ministral-3-3b-instruct" + model_id: "unsloth/Ministral-3-3B-Instruct-2512-GGUF" + quant: "IQ4_XS" + revision: "main" + - name: "qwen2.5-coder-1.5b-instruct" + model_id: "Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF" + quant: "Q4_K_M" + revision: "main" + - name: "qwen2.5-coder-3b-instruct" + model_id: "unsloth/Qwen2.5-Coder-3B-Instruct-128K-GGUF" + quant: "Q4_K_M" + revision: "main" + +llama_server_host: "0.0.0.0" + +llama_server_windows_sources: + - 10.19.4.136 + +llama_server_args_extra: + [ + "-t", + 8, + "-np", + 1, + "-c", + 8192, + "-b", + 1024, + "-ub", + 512, + "-fa", + "on", + "-ctk", + "q4_0", + "-ctv", + "q4_0", + "-cram", + 0, + "-fit", + "off", + "--mmap", + ] + +llama_server_windows_backend: "vulkan" diff --git a/ansible/host_vars/gpu-01.lab.alexpires.me.yml b/ansible/host_vars/gpu-01.lab.alexpires.me.yml index 1607131..2f929f2 100644 --- a/ansible/host_vars/gpu-01.lab.alexpires.me.yml +++ b/ansible/host_vars/gpu-01.lab.alexpires.me.yml @@ -168,6 +168,37 @@ llama_server_models: quant: "IQ4_XS" revision: "main" +llama_server_args_extra: + [ + "-t", + 10, + "-np", + 1, + "-b", + 1024, + "-ub", + 512, + "-fa", + "on", + "-ctk", + "q4_0", + "-ctv", + "q8_0", + "-cram", + -1, + "-sm", + "layer", + "-dev", + "Vulkan1,Vulkan2", + "-ts", + "8,12", + "-fit", + "off", + "-mg", + 1, + "--mmap", + ] + # # Optional global model preset options applied under [*] in models.ini. # # Keys map to llama.cpp CLI args without leading dashes, short args, or LLAMA_ARG_* env names. # llama_preset_global: diff --git a/ansible/roles/llama_server/defaults/main.yml b/ansible/roles/llama_server/defaults/main.yml index bd295ab..b29f5e5 100644 --- a/ansible/roles/llama_server/defaults/main.yml +++ b/ansible/roles/llama_server/defaults/main.yml @@ -9,8 +9,10 @@ llama_server_tag: "server-vulkan" llama_server_extra_groups: "users,video" llama_server_devices: [] llama_server_env: {} -llama_server_port: 8080 +llama_server_port: 11434 llama_server_preset_global: {} +llama_server_host: "127.0.0.1" +llama_server_argv_extra: [] # Llama Exporter configuration llama_server_exporter_enabled: true @@ -24,3 +26,29 @@ llama_server_exporter_scrape_timeout: 5 # Llama.cpp scrape targets llama_server_exporter_targets: [] + +# Windows-specific defaults +llama_server_windows_install_dir: "C:\\llama_server" +llama_server_windows_user: "llama" +llama_server_windows_working_dir: "C:\\llama_server" +llama_server_windows_version: "b9442" +llama_server_windows_backend: "cuda" +llama_server_cuda_version: "13.3" +llama_server_windows_nssm_dir: "C:\\nssm" +llama_server_windows_nssm_version: "2.24" +llama_server_windows_nssm_url: "https://nssm.cc/release/nssm-{{ llama_server_windows_nssm_version }}.zip" + +# Llama-server binary defaults for Windows +llama_server_windows_bin_name: "llama-server.exe" + +# Windows Python defaults +llama_server_windows_python_version: "3.12.10" +llama_server_windows_python_path: "C:\\Program Files\\Python{{ llama_server_windows_python_version | regex_replace('^(\\d+)\\.(\\d+)\\..*$', '\\1\\2') }}\\python.exe" +llama_server_windows_vcredist_url: "https://aka.ms/vs/17/release/vc_redist.x64.exe" + +# Windows Exporter defaults +llama_server_windows_exporter_enabled: true +llama_server_windows_exporter_port: 9550 +llama_server_windows_exporter_interval: 15 +llama_server_windows_exporter_timeout: 5 +llama_server_windows_exporter_targets: [] diff --git a/ansible/roles/llama_server/files/scripts/llama_hf_sync_windows.py b/ansible/roles/llama_server/files/scripts/llama_hf_sync_windows.py new file mode 100644 index 0000000..3129af3 --- /dev/null +++ b/ansible/roles/llama_server/files/scripts/llama_hf_sync_windows.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Synchronize llama.cpp models from Hugging Face into a managed local directory (Windows-compatible).""" + +from __future__ import annotations + +import argparse +import configparser +import json +import os +import re +import shutil +import sys +from pathlib import Path +from typing import Any + +from huggingface_hub import HfApi, snapshot_download + + +def _sanitize_slug(value: str) -> str: + slug = re.sub(r"[^a-zA-Z0-9._-]+", "-", value.strip()) + slug = slug.strip("-._") + return slug.lower() or "model" + + +def _resolve_repo_file(api: HfApi, model: dict[str, Any]) -> str: + repo_files = api.list_repo_files(repo_id=model["model_id"], revision=model["revision"], repo_type="model") + + hf_file = model.get("hf_file") + if hf_file: + if hf_file in repo_files: + return hf_file + raise RuntimeError( + f"Requested hf_file '{hf_file}' was not found in repo " + f"{model['model_id']}@{model['revision']}" + ) + + quant_upper = (model.get("quant") or "").upper() + ggufs = [item for item in repo_files if item.lower().endswith(".gguf")] + matches = [item for item in ggufs if quant_upper in Path(item).name.upper()] + if matches: + return sorted(matches, key=lambda name: (len(Path(name).name), name))[0] + + raise RuntimeError( + f"No GGUF file containing quant '{model.get('quant')}' found in " + f"repo {model['model_id']}@{model['revision']}. Set hf_file explicitly if needed." + ) + + +def _load_models(models_file: Path) -> list[dict[str, Any]]: + data = json.loads(models_file.read_text(encoding="utf-8")) + if not isinstance(data, list): + raise RuntimeError("models file must contain a JSON array") + + normalized: list[dict[str, Any]] = [] + for item in data: + if not isinstance(item, dict): + raise RuntimeError("each model entry must be an object") + if item.get("enable", True) is False: + continue + + model_id = str(item.get("model_id", "")).strip() + quant = str(item.get("quant", "")).strip() + revision = str(item.get("revision", "")).strip() + hf_file = str(item.get("hf_file", "")).strip() or None + name = str(item.get("name", "")).strip() + preset = item.get("preset", {}) + + if preset is None: + preset = {} + if not isinstance(preset, dict): + raise RuntimeError(f"preset must be an object for model '{name or model_id or 'unknown'}'") + + if not name: + raise RuntimeError(f"name is required for model '{model_id or 'unknown'}'") + if not model_id: + raise RuntimeError("model_id is required for all models") + if not quant and not hf_file: + raise RuntimeError(f"quant or hf_file is required for model '{model_id}'") + if not revision: + raise RuntimeError(f"revision is required for model '{model_id}'") + + normalized.append( + { + "model_id": model_id, + "quant": quant, + "revision": revision, + "hf_file": hf_file, + "name": name, + "preset": preset, + } + ) + + return normalized + + +def _load_preset_global(preset_global_file: Path | None) -> dict[str, Any]: + if preset_global_file is None: + return {} + + if not preset_global_file.exists(): + return {} + + data = json.loads(preset_global_file.read_text(encoding="utf-8")) + if data is None: + return {} + if not isinstance(data, dict): + raise RuntimeError("global preset options must be a JSON object") + return data + + +def _to_preset_value(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float, str)): + return str(value) + return json.dumps(value, separators=(",", ":")) + + +def _normalize_preset_options(options: dict[str, Any], scope: str) -> dict[str, str]: + normalized: dict[str, str] = {} + for key, value in options.items(): + key_str = str(key).strip() + if not key_str: + raise RuntimeError(f"{scope} preset option keys must be non-empty") + if value is None: + continue + normalized[key_str] = _to_preset_value(value) + return normalized + + +def _download_model(model: dict[str, Any], target_dir: Path, repo_file: str, dry_run: bool) -> None: + if dry_run: + return + + snapshot_download( + repo_id=model["model_id"], + revision=model["revision"], + local_dir=str(target_dir), + allow_patterns=[repo_file], + local_dir_use_symlinks=False, + ) + + +def _write_preset(preset_file: Path, entries: list[dict[str, Any]], global_options: dict[str, Any]) -> None: + parser = configparser.ConfigParser(interpolation=None) + parser.optionxform = str + parser["*"] = _normalize_preset_options(global_options, "global") + + for entry in entries: + model_options = _normalize_preset_options(entry.get("preset", {}), f"model '{entry['name']}'") + model_options["model"] = entry["container_model_path"] + parser[entry["name"]] = model_options + + preset_file.parent.mkdir(parents=True, exist_ok=True) + with preset_file.open("w", encoding="utf-8") as fh: + # Router preset format supports top-level version key. + fh.write("version = 1\n\n") + parser.write(fh) + + +def _prune_unmanaged(managed_dir: Path, links_dir: Path, keep_slugs: set[str], dry_run: bool) -> tuple[list[str], list[str]]: + pruned_dirs: list[str] = [] + pruned_links: list[str] = [] + + if managed_dir.exists(): + for child in managed_dir.iterdir(): + if not child.is_dir(): + continue + if child.name == ".router": + continue + if child.name not in keep_slugs: + pruned_dirs.append(child.name) + if not dry_run: + shutil.rmtree(child) + + if links_dir.exists(): + for child in links_dir.iterdir(): + if child.suffix != ".gguf": + continue + slug = child.stem + if slug not in keep_slugs: + pruned_links.append(child.name) + if not dry_run: + child.unlink(missing_ok=True) + + return pruned_dirs, pruned_links + + +def _create_hard_link(target: Path, link_path: Path) -> None: + """Create a hard link on Windows, falling back to copy if hard links are not supported.""" + if link_path.exists() or link_path.is_symlink(): + link_path.unlink() + + try: + # On Windows, hard links require the source to be on the same volume + link_path.hardlink_to(target) + except (OSError, NotImplementedError): + # Fall back to copy if hard links are not supported (different volumes, permissions, etc.) + shutil.copy2(str(target), str(link_path)) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Sync Hugging Face GGUF models for llama.cpp router mode (Windows)") + parser.add_argument("--models-file", required=True) + parser.add_argument("--managed-dir", required=True) + parser.add_argument("--links-dir", required=True) + parser.add_argument("--manifest-file", required=True) + parser.add_argument("--preset-file", required=True) + parser.add_argument("--preset-global-file") + parser.add_argument("--container-links-dir", required=True) + parser.add_argument("--prune", action="store_true") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + models_file = Path(args.models_file) + managed_dir = Path(args.managed_dir) + links_dir = Path(args.links_dir) + manifest_file = Path(args.manifest_file) + preset_file = Path(args.preset_file) + preset_global_file = Path(args.preset_global_file) if args.preset_global_file else None + + managed_dir.mkdir(parents=True, exist_ok=True) + links_dir.mkdir(parents=True, exist_ok=True) + manifest_file.parent.mkdir(parents=True, exist_ok=True) + preset_file.parent.mkdir(parents=True, exist_ok=True) + + models = _load_models(models_file) + global_options = _load_preset_global(preset_global_file) + api = HfApi() + entries: list[dict[str, Any]] = [] + + for model in models: + slug = _sanitize_slug(model["name"]) + target_dir = managed_dir / slug + target_dir.mkdir(parents=True, exist_ok=True) + repo_file = _resolve_repo_file(api, model) + + _download_model(model, target_dir, repo_file, args.dry_run) + selected_file = target_dir / repo_file + if not args.dry_run and not selected_file.exists(): + raise RuntimeError(f"Expected downloaded file not found: {selected_file}") + + link_path = links_dir / f"{slug}.gguf" + container_model_path = f"{args.container_links_dir.rstrip('/')}/{slug}.gguf" + + if not args.dry_run: + _create_hard_link(selected_file, link_path) + + entries.append( + { + "name": model["name"], + "slug": slug, + "model_id": model["model_id"], + "revision": model["revision"], + "quant": model["quant"], + "hf_file": model.get("hf_file"), + "preset": model.get("preset", {}), + "repo_file": repo_file, + "selected_file": str(selected_file), + "link_path": str(link_path), + "container_model_path": container_model_path, + } + ) + + pruned_dirs: list[str] = [] + pruned_links: list[str] = [] + if args.prune: + keep_slugs = {entry["slug"] for entry in entries} + pruned_dirs, pruned_links = _prune_unmanaged(managed_dir, links_dir, keep_slugs, args.dry_run) + + result = { + "models": entries, + "dry_run": args.dry_run, + "prune": args.prune, + "pruned_dirs": pruned_dirs, + "pruned_links": pruned_links, + } + + if not args.dry_run: + manifest_file.write_text(json.dumps(result, indent=2), encoding="utf-8") + _write_preset(preset_file, entries, global_options) + + print(json.dumps(result)) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: + print(f"ERROR: {exc}", file=sys.stderr) + raise SystemExit(1) diff --git a/ansible/roles/llama_server/meta/main.yml b/ansible/roles/llama_server/meta/main.yml index 3780fd6..c7f5786 100644 --- a/ansible/roles/llama_server/meta/main.yml +++ b/ansible/roles/llama_server/meta/main.yml @@ -11,3 +11,7 @@ galaxy_info: - name: EL versions: - "9" + - name: Windows + versions: + - "2019" + - "2022" diff --git a/ansible/roles/llama_server/tasks/llama_exporter.yml b/ansible/roles/llama_server/tasks/el/llama_exporter.yml similarity index 100% rename from ansible/roles/llama_server/tasks/llama_exporter.yml rename to ansible/roles/llama_server/tasks/el/llama_exporter.yml diff --git a/ansible/roles/llama_server/tasks/el/main.yml b/ansible/roles/llama_server/tasks/el/main.yml new file mode 100644 index 0000000..089d719 --- /dev/null +++ b/ansible/roles/llama_server/tasks/el/main.yml @@ -0,0 +1,336 @@ +--- +- name: Validate llama_server_models schema + ansible.builtin.assert: + that: + - llama_server_models is iterable + fail_msg: "llama_server_models must be a list" + +- name: Create group + become: true + ansible.builtin.group: + name: "{{ llama_server_group }}" + system: true + +- name: Create user + become: true + ansible.builtin.user: + name: "{{ llama_server_user }}" + groups: "{{ podman_extra_groups | default([]) }}" + shell: /bin/bash + home: "{{ llama_server_home }}" + group: "{{ llama_server_group }}" + +- name: Disable password login for user + ansible.builtin.command: passwd -d "{{ llama_server_user }}" + become: true + changed_when: false + +- name: Create user home + become: true + ansible.builtin.file: + path: "{{ llama_server_home }}" + state: directory + owner: "{{ llama_server_user }}" + group: "{{ llama_server_group }}" + mode: "0750" + +- name: Set fact podman_binary (Debian family) + ansible.builtin.set_fact: + podman_binary: /usr/bin/podman + when: ansible_os_family == 'Debian' + +- name: Set fact podman_binary (Red Hat family) + ansible.builtin.set_fact: + podman_binary: /usr/sbin/podman + when: ansible_os_family == 'RedHat' + +- name: Create required folders + become: true + ansible.builtin.file: + path: "{{ llama_server_home }}/{{ item }}" + state: directory + owner: "{{ llama_server_user }}" + group: "{{ llama_server_group }}" + mode: "0750" + loop: + - models + - .cache + - .cache/llama.cpp + +- name: "Add SSH Authorized key" + become: true + ansible.posix.authorized_key: + user: "{{ llama_server_user }}" + key: "{{ llama_server_pubkey }}" + state: "{{ 'present' if llama_server_pubkey is defined and llama_server_pubkey != '' else 'absent' }}" + when: llama_server_pubkey is defined and llama_server_pubkey != "" + +- name: SELinux tasks (RedHat only) + when: ansible_os_family == 'RedHat' + become: true + block: + - name: Persistently set SELinux context + community.general.sefcontext: + target: "{{ llama_server_home }}(/.*)?" + setype: user_home_dir_t + state: present + + - name: Persistently set SELinux context (ssh authorized keys) + community.general.sefcontext: + target: "{{ llama_server_home }}/.ssh/authorized_keys" + setype: ssh_home_t + state: present + when: llama_server_pubkey is defined and llama_server_pubkey != "" + + - name: Apply SELinux context + ansible.builtin.command: restorecon -Rv {{ llama_server_home }} + changed_when: false + +- name: Enable sudo access to llama user (temporary) + become: true + ansible.builtin.lineinfile: + path: "/etc/sudoers.d/ansible_{{ llama_server_user }}_tmp" + create: true + mode: "0440" + line: "{{ ansible_facts['user_id'] }} ALL=({{ llama_server_user }}) NOPASSWD: ALL" + validate: "visudo -cf %s" + +- name: Enable lingering for llama user + become: true + ansible.builtin.command: loginctl enable-linger {{ llama_server_user }} + register: linger_status + changed_when: >- + 'created' in linger_status.stdout or + (linger_status.rc == 0 and not + ('linger file already exists' in linger_status.stderr or + 'linger file does not exist' in linger_status.stderr)) + failed_when: + - linger_status.rc != 0 and not + ('linger file already exists' in linger_status.stderr) + +- name: Get current user's UID + ansible.builtin.command: id -u {{ llama_server_user }} + changed_when: false + register: uid + +- name: Get current user's GID + ansible.builtin.command: id -g {{ llama_server_user }} + changed_when: false + register: gid + +- name: Run tasks as podman user + become: true + become_user: "{{ llama_server_user }}" + block: + - name: Pull the requested images + containers.podman.podman_image: + name: "{{ llama_server_image }}" + tag: "{{ llama_server_tag }}" + state: present + + - name: Create pods + containers.podman.podman_container: + name: "llama.cpp" + image: "{{ llama_server_image }}:{{ llama_server_tag }}" + state: started + device: "{{ llama_server_devices | default(omit) }}" + env: "{{ llama_server_env | default(omit) }}" + ports: "0.0.0.0:{{ llama_server_port }}:8080/tcp" + volumes: + - "{{ llama_server_home }}/models:/models:Z" + - "{{ llama_server_home }}/.cache:/app/.cache:Z" + command: >- + {{ + [ + '--metrics', + '--models-preset', + '/models/.router/models.ini', + '--models-max', + (llama_server_models_max | string) + ] + + (llama_server_argv_extra | default([])) + }} + + cmd_args: + - "--userns=keep-id" + - "--security-opt=label=disable" + +- name: Create systemd service file for pods + become: true + ansible.builtin.template: + src: podman.service.j2 + dest: "/etc/systemd/system/podman-llama.cpp.service" + owner: root + group: root + mode: "0644" + +- name: Reload systemd daemon + become: true + ansible.builtin.systemd: + daemon_reload: true + +- name: Enable containers at boot + become: true + ansible.builtin.systemd: + name: "podman-llama.cpp" + enabled: true + state: "started" + +- name: Install Python packages required for llama server model sync + become: true + ansible.builtin.package: + name: "{{ __llama_sync_system_packages__ }}" + state: present + vars: + __llama_sync_system_packages__: >- + {{ + { + 'Debian': ['python3-pip', 'python3-packaging', 'python3-venv'], + 'RedHat': ['python3-pip', 'python3-packaging'], + 'Archlinux': ['python-pip', 'python-packaging'] + }[ansible_os_family] + }} + +- name: Install Hugging Face hub client in llama server sync virtualenv + become: true + become_user: "{{ llama_server_user }}" + ansible.builtin.pip: + name: + - pip + - setuptools + - packaging + - huggingface_hub>=0.32.0 + state: present + virtualenv: "{{ llama_server_home }}/.router/.venv" + virtualenv_command: "python3 -m venv" + +- name: Create llama server router model directories + become: true + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + owner: "{{ llama_server_user }}" + group: "{{ llama_server_group }}" + mode: "{{ item.mode }}" + loop: + - { path: "{{ llama_server_home }}/models", mode: "0750" } + - { path: "{{ llama_server_home }}/models/managed", mode: "0750" } + - { path: "{{ llama_server_home }}/models/managed-links", mode: "0750" } + - { path: "{{ llama_server_home }}/models/.router", mode: "0750" } + +- name: Copy llama server Hugging Face sync helper + become: true + ansible.builtin.copy: + src: scripts/llama_hf_sync.py + dest: "{{ llama_server_home }}/models/.router/llama_hf_sync.py" + owner: "{{ llama_server_user }}" + group: "{{ llama_server_group }}" + mode: "0750" + +- name: Write desired llama server model manifest input + become: true + ansible.builtin.copy: + content: "{{ llama_server_models | to_nice_json }}" + dest: "{{ llama_server_home }}/models/.router/desired-models.json" + owner: "{{ llama_server_user }}" + group: "{{ llama_server_group }}" + mode: "0640" + +- name: Write global llama server preset options + become: true + ansible.builtin.copy: + content: "{{ llama_server_preset_global | to_nice_json }}" + dest: "{{ llama_server_home }}/models/.router/preset-global.json" + owner: "{{ llama_server_user }}" + group: "{{ llama_server_group }}" + mode: "0640" + +- name: Set arguments for llama server model sync + ansible.builtin.set_fact: + __llama_preset_file__: + - "{{ llama_server_home }}/.router/.venv/bin/python" + - "{{ llama_server_home }}/models/.router/llama_hf_sync.py" + - "--models-file" + - "{{ llama_server_home }}/models/.router/desired-models.json" + - "--managed-dir" + - "{{ llama_server_home }}/models/managed" + - "--links-dir" + - "{{ llama_server_home }}/models/managed-links" + - "--manifest-file" + - "{{ llama_server_home }}/models/.router/manifest.json" + - "--preset-file" + - "{{ llama_server_home }}/models/.router/models.ini" + - "--preset-global-file" + - "{{ llama_server_home }}/models/.router/preset-global.json" + - "--container-links-dir" + - "/models/managed-links" + +- name: Run llama server model sync from Hugging Face + vars: + __llama_sync_argv__: >- + {{ __llama_preset_file__ + + (['--prune'] if llama_server_models_prune_enabled | default(true) else []) + + (['--dry-run'] if llama_server_models_dry_run | default(false) else []) + }} + become: true + become_user: "{{ llama_server_user }}" + ansible.builtin.command: + argv: "{{ __llama_sync_argv__ }}" + environment: + HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') | default('', true) }}" + register: __llama_sync_output__ + changed_when: false + +- name: Parse llama server sync output + ansible.builtin.set_fact: + __llama_sync_result__: "{{ __llama_sync_output__.stdout | from_json }}" + +- name: Stat llama router preset file (when not in dry-run mode) + become: true + become_user: "{{ llama_server_user }}" + ansible.builtin.stat: + path: "{{ llama_server_home }}/models/.router/models.ini" + register: __llama_preset_stat__ + when: not (llama_server_models_dry_run | default(false)) + +- name: Ensure router preset file exists when not in dry-run mode + ansible.builtin.assert: + that: + - __llama_preset_stat__.stat.exists + fail_msg: "llama router preset file was not created" + when: not (llama_server_models_dry_run | default(false)) + +- name: Query llama router model list + ansible.builtin.uri: + url: "http://127.0.0.1:{{ llama_server_port | default(11434) }}/models?reload=1" + method: GET + status_code: 200 + register: __llama_router_models__ + changed_when: false + when: + - llama_server_models_sync_enabled | default(true) + - not (llama_server_models_dry_run | default(false)) + +- name: Validate expected model aliases are available in router + ansible.builtin.assert: + that: + - item.name in (__llama_router_models__.json.data | map(attribute='id') | list) + fail_msg: "Model name '{{ item.name }}' not present in llama router /models output" + loop: "{{ __llama_sync_result__.models | default([]) }}" + loop_control: + label: "{{ item.name }}" + when: + - llama_server_models_sync_enabled | default(true) + - not (llama_server_models_dry_run | default(false)) + +- name: Include Llama Exporter setup + when: llama_server_exporter_enabled | default(true) | bool + ansible.builtin.include_tasks: llama_exporter.yml + tags: + - roles::llama_server::llama_exporter + +- name: Remove temporary sudo access + become: true + ansible.builtin.file: + path: "/etc/sudoers.d/ansible_{{ llama_server_user }}_tmp" + state: absent diff --git a/ansible/roles/llama_server/tasks/main.yml b/ansible/roles/llama_server/tasks/main.yml index 3e0e3dd..8475123 100644 --- a/ansible/roles/llama_server/tasks/main.yml +++ b/ansible/roles/llama_server/tasks/main.yml @@ -1,358 +1,17 @@ --- -- name: Validate llama_server_models schema - ansible.builtin.assert: - that: - - llama_server_models is iterable - fail_msg: "llama_server_models must be a list" +# --- EL-specific setup (podman + systemd) --- -- name: Create group - become: true - ansible.builtin.group: - name: "{{ llama_server_group }}" - system: true - -- name: Create user - become: true - ansible.builtin.user: - name: "{{ llama_server_user }}" - groups: "{{ podman_extra_groups | default([]) }}" - shell: /bin/bash - home: "{{ llama_server_home }}" - group: "{{ llama_server_group }}" - -- name: Disable password login for user - ansible.builtin.command: passwd -d "{{ llama_server_user }}" - become: true - changed_when: false - -- name: Create user home - become: true - ansible.builtin.file: - path: "{{ llama_server_home }}" - state: directory - owner: "{{ llama_server_user }}" - group: "{{ llama_server_group }}" - mode: "0750" - -- name: Set fact podman_binary (Debian family) - ansible.builtin.set_fact: - podman_binary: /usr/bin/podman - when: ansible_os_family == 'Debian' - -- name: Set fact podman_binary (Red Hat family) - ansible.builtin.set_fact: - podman_binary: /usr/sbin/podman +- name: Run EL container setup when: ansible_os_family == 'RedHat' - -- name: Create required folders - become: true - ansible.builtin.file: - path: "{{ llama_server_home }}/{{ item }}" - state: directory - owner: "{{ llama_server_user }}" - group: "{{ llama_server_group }}" - mode: "0750" - loop: - - models - - .cache - - .cache/llama.cpp - -- name: "Add SSH Authorized key" - become: true - ansible.posix.authorized_key: - user: "{{ llama_server_user }}" - key: "{{ llama_server_pubkey }}" - state: "{{ 'present' if llama_server_pubkey is defined and llama_server_pubkey != '' else 'absent' }}" - when: llama_server_pubkey is defined and llama_server_pubkey != "" - -- name: SELinux tasks (RedHat only) - when: ansible_os_family == 'RedHat' - become: true - block: - - name: Persistently set SELinux context - community.general.sefcontext: - target: "{{ llama_server_home }}(/.*)?" - setype: user_home_dir_t - state: present - - - name: Persistently set SELinux context (ssh authorized keys) - community.general.sefcontext: - target: "{{ llama_server_home }}/.ssh/authorized_keys" - setype: ssh_home_t - state: present - when: llama_server_pubkey is defined and llama_server_pubkey != "" - - - name: Apply SELinux context - ansible.builtin.command: restorecon -Rv {{ llama_server_home }} - changed_when: false - -- name: Enable sudo access to llama user (temporary) - become: true - ansible.builtin.lineinfile: - path: "/etc/sudoers.d/ansible_{{ llama_server_user }}_tmp" - create: true - mode: "0440" - line: "{{ ansible_facts['user_id'] }} ALL=({{ llama_server_user }}) NOPASSWD: ALL" - validate: "visudo -cf %s" - -- name: Enable lingering for llama user - become: true - ansible.builtin.command: loginctl enable-linger {{ llama_server_user }} - register: linger_status - changed_when: >- - 'created' in linger_status.stdout or - (linger_status.rc == 0 and not - ('linger file already exists' in linger_status.stderr or - 'linger file does not exist' in linger_status.stderr)) - failed_when: - - linger_status.rc != 0 and not - ('linger file already exists' in linger_status.stderr) - -- name: Get current user's UID - ansible.builtin.command: id -u {{ llama_server_user }} - changed_when: false - register: uid - -- name: Get current user's GID - ansible.builtin.command: id -g {{ llama_server_user }} - changed_when: false - register: gid - -- name: Run tasks as podman user - become: true - become_user: "{{ llama_server_user }}" - block: - - name: Pull the requested images - containers.podman.podman_image: - name: "{{ llama_server_image }}" - tag: "{{ llama_server_tag }}" - state: present - - - name: Create pods - containers.podman.podman_container: - name: "llama.cpp" - image: "{{ llama_server_image }}:{{ llama_server_tag }}" - state: started - device: "{{ llama_server_devices | default(omit) }}" - env: "{{ llama_server_env | default(omit) }}" - ports: "0.0.0.0:{{ llama_server_port }}:8080/tcp" - volumes: - - "{{ llama_server_home }}/models:/models:Z" - - "{{ llama_server_home }}/.cache:/app/.cache:Z" - command: - - "-t" - - "{{ llama_server_argv_threads | default(10) }}" - - "-np" - - "{{ llama_server_argv_parallel | default(1) }}" - - "-b" - - "{{ llama_server_argv_batch_size | default(1024) }}" - - "-ub" - - "{{ llama_server_argv_ubatch_size | default(512) }}" - - "-fa" - - "{{ llama_server_argv_flash_attn | default('on') }}" - - "-ctk" - - "{{ llama_server_argv_cache_type_k | default('q4_0') }}" - - "-ctv" - - "{{ llama_server_argv_cache_type_v | default('q8_0') }}" - - "-cram" - - "{{ llama_server_argv_cache_reuse | default(-1) }}" - - "-sm" - - "{{ llama_server_argv_split_mode | default('layer') }}" - - "-dev" - - "{{ llama_server_argv_devices | default('Vulkan1,Vulkan2') }}" - - "-ts" - - "{{ llama_server_argv_tensor_split | default('8,12') }}" - - "-fit" - - "{{ llama_server_argv_fit | default('off') }}" - - "-mg" - - "{{ llama_server_argv_main_gpu | default(1) }}" - - "--mmap" - - "--metrics" - - "--models-preset" - - "/models/.router/models.ini" - - "--models-max" - - "{{ llama_server_models_max }}" - - cmd_args: - - "--userns=keep-id" - - "--security-opt=label=disable" - -- name: Create systemd service file for pods - become: true - ansible.builtin.template: - src: podman.service.j2 - dest: "/etc/systemd/system/podman-llama.cpp.service" - owner: root - group: root - mode: "0644" - -- name: Reload systemd daemon - become: true - ansible.builtin.systemd: - daemon_reload: true - -- name: Enable containers at boot - become: true - ansible.builtin.systemd: - name: "podman-llama.cpp" - enabled: true - state: "started" - -- name: Install Python packages required for llama server model sync - become: true - ansible.builtin.package: - name: "{{ __llama_sync_system_packages__ }}" - state: present - vars: - __llama_sync_system_packages__: >- - {{ - { - 'Debian': ['python3-pip', 'python3-packaging', 'python3-venv'], - 'RedHat': ['python3-pip', 'python3-packaging'], - 'Archlinux': ['python-pip', 'python-packaging'] - }[ansible_os_family] - }} - -- name: Install Hugging Face hub client in llama server sync virtualenv - become: true - become_user: "{{ llama_server_user }}" - ansible.builtin.pip: - name: - - pip - - setuptools - - packaging - - huggingface_hub>=0.32.0 - state: present - virtualenv: "{{ llama_server_home }}/.router/.venv" - virtualenv_command: "python3 -m venv" - -- name: Create llama server router model directories - become: true - ansible.builtin.file: - path: "{{ item.path }}" - state: directory - owner: "{{ llama_server_user }}" - group: "{{ llama_server_group }}" - mode: "{{ item.mode }}" - loop: - - { path: "{{ llama_server_home }}/models", mode: "0750" } - - { path: "{{ llama_server_home }}/models/managed", mode: "0750" } - - { path: "{{ llama_server_home }}/models/managed-links", mode: "0750" } - - { path: "{{ llama_server_home }}/models/.router", mode: "0750" } - -- name: Copy llama server Hugging Face sync helper - become: true - ansible.builtin.copy: - src: scripts/llama_hf_sync.py - dest: "{{ llama_server_home }}/models/.router/llama_hf_sync.py" - owner: "{{ llama_server_user }}" - group: "{{ llama_server_group }}" - mode: "0750" - -- name: Write desired llama server model manifest input - become: true - ansible.builtin.copy: - content: "{{ llama_server_models | to_nice_json }}" - dest: "{{ llama_server_home }}/models/.router/desired-models.json" - owner: "{{ llama_server_user }}" - group: "{{ llama_server_group }}" - mode: "0640" - -- name: Write global llama server preset options - become: true - ansible.builtin.copy: - content: "{{ llama_server_preset_global | to_nice_json }}" - dest: "{{ llama_server_home }}/models/.router/preset-global.json" - owner: "{{ llama_server_user }}" - group: "{{ llama_server_group }}" - mode: "0640" - -- name: Set arguments for llama server model sync - ansible.builtin.set_fact: - __llama_preset_file__: - - "{{ llama_server_home }}/.router/.venv/bin/python" - - "{{ llama_server_home }}/models/.router/llama_hf_sync.py" - - "--models-file" - - "{{ llama_server_home }}/models/.router/desired-models.json" - - "--managed-dir" - - "{{ llama_server_home }}/models/managed" - - "--links-dir" - - "{{ llama_server_home }}/models/managed-links" - - "--manifest-file" - - "{{ llama_server_home }}/models/.router/manifest.json" - - "--preset-file" - - "{{ llama_server_home }}/models/.router/models.ini" - - "--preset-global-file" - - "{{ llama_server_home }}/models/.router/preset-global.json" - - "--container-links-dir" - - "/models/managed-links" - -- name: Run llama server model sync from Hugging Face - vars: - __llama_sync_argv__: >- - {{ __llama_preset_file__ - + (['--prune'] if llama_server_models_prune_enabled | default(true) else []) - + (['--dry-run'] if llama_server_models_dry_run | default(false) else []) - }} - become: true - become_user: "{{ llama_server_user }}" - ansible.builtin.command: - argv: "{{ __llama_sync_argv__ }}" - environment: - HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') | default('', true) }}" - register: __llama_sync_output__ - changed_when: false - -- name: Parse llama server sync output - ansible.builtin.set_fact: - __llama_sync_result__: "{{ __llama_sync_output__.stdout | from_json }}" - -- name: Stat llama router preset file (when not in dry-run mode) - become: true - become_user: "{{ llama_server_user }}" - ansible.builtin.stat: - path: "{{ llama_server_home }}/models/.router/models.ini" - register: __llama_preset_stat__ - when: not (llama_server_models_dry_run | default(false)) - -- name: Ensure router preset file exists when not in dry-run mode - ansible.builtin.assert: - that: - - __llama_preset_stat__.stat.exists - fail_msg: "llama router preset file was not created" - when: not (llama_server_models_dry_run | default(false)) - -- name: Query llama router model list - ansible.builtin.uri: - url: "http://127.0.0.1:{{ llama_server_port | default(8080) }}/models?reload=1" - method: GET - status_code: 200 - register: __llama_router_models__ - changed_when: false - when: - - llama_server_models_sync_enabled | default(true) - - not (llama_server_models_dry_run | default(false)) - -- name: Validate expected model aliases are available in router - ansible.builtin.assert: - that: - - item.name in (__llama_router_models__.json.data | map(attribute='id') | list) - fail_msg: "Model name '{{ item.name }}' not present in llama router /models output" - loop: "{{ __llama_sync_result__.models | default([]) }}" - loop_control: - label: "{{ item.name }}" - when: - - llama_server_models_sync_enabled | default(true) - - not (llama_server_models_dry_run | default(false)) - -- name: Include Llama Exporter setup - when: llama_server_exporter_enabled | default(true) | bool - ansible.builtin.include_tasks: llama_exporter.yml + ansible.builtin.include_tasks: + file: el/main.yml tags: - - roles::llama_server::llama_exporter + - roles::llama_server::el -- name: Remove temporary sudo access - become: true - ansible.builtin.file: - path: "/etc/sudoers.d/ansible_{{ llama_server_user }}_tmp" - state: absent +# --- Windows-specific setup (NSSM) --- +- name: Run Windows setup + when: ansible_os_family == 'Windows' + ansible.builtin.include_tasks: + file: windows/main.yml + tags: + - roles::llama_server::windows diff --git a/ansible/roles/llama_server/tasks/windows/llama_exporter.yml b/ansible/roles/llama_server/tasks/windows/llama_exporter.yml new file mode 100644 index 0000000..641f5fb --- /dev/null +++ b/ansible/roles/llama_server/tasks/windows/llama_exporter.yml @@ -0,0 +1,163 @@ +--- +# Windows Exporter tasks for llama_server role + +- name: Define Windows Exporter paths + ansible.builtin.set_fact: + __exporter_install_dir: "{{ llama_server_windows_install_dir }}\\exporter" + __exporter_script: "{{ llama_server_windows_install_dir }}\\exporter\\llama_exporter.py" + __exporter_venv_path: "{{ llama_server_windows_install_dir }}\\exporter\\.venv" + __exporter_venv_python: "{{ llama_server_windows_install_dir }}\\exporter\\.venv\\Scripts\\python.exe" + __exporter_env: "LLAMA_EXPORTER_PORT={{ llama_server_windows_exporter_port | default(9550) }}" + __exporter_interval: "{{ llama_server_windows_exporter_interval | default(15) }}" + __exporter_timeout: "{{ llama_server_windows_exporter_timeout | default(5) }}" + +- name: Create exporter directories + ansible.windows.win_file: + path: "{{ item }}" + state: directory + loop: + - "{{ llama_server_windows_install_dir }}\\exporter" + +- name: Copy Windows Exporter script + ansible.windows.win_copy: + src: scripts/llama_exporter.py + dest: "{{ __exporter_script }}" + +- name: Install Python packages for Windows Exporter + ansible.windows.win_stat: + path: "{{ __exporter_venv_python }}" + register: __exporter_venv_python_check + +- name: Create exporter virtualenv + ansible.windows.win_command: + argv: + - "{{ llama_server_windows_python_path }}" + - -m + - venv + - "{{ __exporter_venv_path }}" + changed_when: false + when: not __exporter_venv_python_check.stat.exists + +- name: Install Python packages for Windows Exporter + ansible.windows.win_command: + argv: + - "{{ __exporter_venv_python }}" + - -m + - pip + - install + - --upgrade + - pip + - setuptools + changed_when: false + +- name: Remove existing exporter service if present + ansible.windows.win_service: + name: llama-exporter + state: absent + register: __exporter_remove_service + failed_when: false + +- name: Set exporter targets environment variable if provided + ansible.builtin.set_fact: + __exporter_targets_env: "LLAMA_TARGETS={{ llama_server_windows_exporter_targets | to_json }}" + when: llama_server_windows_exporter_targets | default([]) | length > 0 + +- name: Install Windows Exporter service via NSSM + ansible.windows.win_command: + argv: + - "{{ __nssm_exe }}" + - install + - llama-exporter + - "{{ __exporter_venv_python }}" + - "{{ __exporter_script }}" + args: + chdir: "{{ __exporter_install_dir }}" + register: __exporter_nssm_install + changed_when: __exporter_nssm_install.rc == 0 + +- name: Set exporter AppDirectory via NSSM + ansible.windows.win_command: + argv: + - "{{ __nssm_exe }}" + - set + - llama-exporter + - AppDirectory + - "{{ __exporter_install_dir }}" + +- name: Configure exporter stdout log via NSSM + ansible.windows.win_command: + argv: + - "{{ __nssm_exe }}" + - set + - llama-exporter + - AppStdout + - "{{ llama_server_windows_install_dir }}\\exporter-service.out.log" + +- name: Configure exporter stderr log via NSSM + ansible.windows.win_command: + argv: + - "{{ __nssm_exe }}" + - set + - llama-exporter + - AppStderr + - "{{ llama_server_windows_install_dir }}\\exporter-service.err.log" + +- name: Set exporter environment variables + ansible.windows.win_command: + argv: + - "{{ __nssm_exe }}" + - set + - llama-exporter + - AppEnvironmentExtra + - "{{ (__exporter_env ~ (('\n' ~ __exporter_targets_env) if (__exporter_targets_env is defined) else '')) }}" + when: + - __exporter_env | default('') != '' + +- name: Set exporter Start mode to AUTO via NSSM + ansible.windows.win_command: + argv: + - "{{ __nssm_exe }}" + - set + - llama-exporter + - Start + - SERVICE_AUTO_START + +- name: Ensure Windows Exporter service is running + ansible.windows.win_service: + name: llama-exporter + start_mode: auto + state: restarted + when: not ansible_check_mode | default(false) + +- name: Wait for exporter to listen on port + ansible.windows.win_wait_for: + port: "{{ llama_server_windows_exporter_port | default(9550) }}" + delay: 5 + timeout: 30 + when: not ansible_check_mode | default(false) + +- name: Set firewall remote IPs when string provided + ansible.builtin.set_fact: + llama_server_windows_remoteip: "{{ llama_server_windows_sources }}" + when: + - llama_server_windows_sources is defined + - llama_server_windows_sources is string + +- name: Set firewall remote IPs when list provided + ansible.builtin.set_fact: + llama_server_windows_remoteip: "{{ llama_server_windows_sources | join(',') }}" + when: + - llama_server_windows_sources is defined + - not (llama_server_windows_sources is string) + +- name: Allow exporter port through Windows Firewall + community.windows.win_firewall_rule: + name: Llama Exporter "{{ llama_server_windows_exporter_port | default(9550) }}" + localport: "{{ llama_server_windows_exporter_port | default(9550) }}" + protocol: tcp + direction: in + action: allow + enabled: true + # Provide one or more IPs/CIDRs in llama_server_windows_sources (e.g. ['203.0.113.10']) + remoteip: "{{ llama_server_windows_remoteip | default(omit) }}" + state: present diff --git a/ansible/roles/llama_server/tasks/windows/main.yml b/ansible/roles/llama_server/tasks/windows/main.yml new file mode 100644 index 0000000..6e568e8 --- /dev/null +++ b/ansible/roles/llama_server/tasks/windows/main.yml @@ -0,0 +1,382 @@ +--- +# Windows-specific setup for llama_server role + +- name: Define Windows paths and versions + ansible.builtin.set_fact: + __llama_user: "{{ llama_server_windows_user }}" + __nssm_exe: "{{ llama_server_windows_nssm_dir }}\\nssm.exe" + __llama_release_asset_name: "{{ 'llama-' ~ llama_server_windows_version ~ '-bin-win-' ~ llama_server_windows_backend ~ (llama_server_windows_backend == 'cuda') | ternary('-' ~ llama_server_cuda_version, '') ~ '-x64.zip' }}" + __llama_release_zip_url: "https://github.com/ggml-org/llama.cpp/releases/download/{{ llama_server_windows_version }}/{{ 'llama-' ~ llama_server_windows_version ~ '-bin-win-' ~ llama_server_windows_backend ~ (llama_server_windows_backend == 'cuda') | ternary('-' ~ llama_server_cuda_version, '') ~ '-x64.zip' }}" + __llama_release_zip_path: "{{ (ansible_env.TEMP | default('C:\\Windows\\Temp')) }}\\{{ 'llama-' ~ llama_server_windows_version ~ '-bin-win-' ~ llama_server_windows_backend ~ (llama_server_windows_backend == 'cuda') | ternary('-' ~ llama_server_cuda_version, '') ~ '-x64.zip' }}" + __llama_release_marker: "{{ llama_server_windows_install_dir }}\\.release-{{ 'llama-' ~ llama_server_windows_version ~ '-bin-win-' ~ llama_server_windows_backend ~ (llama_server_windows_backend == 'cuda') | ternary('-' ~ llama_server_cuda_version, '') ~ '-x64.zip' }}" + __nssm_zip_path: "{{ (ansible_env.TEMP | default('C:\\Windows\\Temp')) }}\\nssm.zip" + __llama_sync_script: "{{ llama_server_windows_working_dir }}\\.router\\llama_hf_sync_windows.py" + __llama_models_json: "{{ llama_server_windows_working_dir }}\\.router\\desired-models.json" + __llama_preset_global_json: "{{ llama_server_windows_working_dir }}\\.router\\preset-global.json" + __llama_managed_dir: "{{ llama_server_windows_working_dir }}\\models\\managed" + __llama_links_dir: "{{ llama_server_windows_working_dir }}\\models\\managed-links" + __llama_router_dir: "{{ llama_server_windows_working_dir }}\\.router" + __llama_manifest: "{{ llama_server_windows_working_dir }}\\.router\\manifest.json" + __llama_preset_file: "{{ llama_server_windows_working_dir }}\\.router\\models.ini" + __llama_venv_path: "{{ llama_server_windows_working_dir }}\\.router\\.venv" + __llama_venv_python: "{{ llama_server_windows_working_dir }}\\.router\\.venv\\Scripts\\python.exe" + +- name: Validate llama_server_models schema + ansible.builtin.assert: + that: + - llama_server_models is iterable + fail_msg: "llama_server_models must be a list" + +- name: Create llama server installation directories + ansible.windows.win_file: + path: "{{ item }}" + state: directory + loop: + - "{{ llama_server_windows_install_dir }}" + - "{{ llama_server_windows_working_dir }}" + - "{{ llama_server_windows_working_dir }}\\models\\managed" + - "{{ llama_server_windows_working_dir }}\\models\\managed-links" + - "{{ llama_server_windows_working_dir }}\\.router" + +- name: Check if Python is available + ansible.windows.win_stat: + path: "{{ llama_server_windows_python_path }}" + register: __llama_python_check + +- name: Download Python installer + ansible.windows.win_get_url: + url: "https://www.python.org/ftp/python/{{ llama_server_windows_python_version }}/python-{{ llama_server_windows_python_version }}-amd64.exe" + dest: "{{ (ansible_env.TEMP | default('C:\\Windows\\Temp')) }}\\python-{{ llama_server_windows_python_version }}-amd64.exe" + force: false + when: not __llama_python_check.stat.exists + +- name: Install Python if not available + ansible.windows.win_package: + path: "{{ (ansible_env.TEMP | default('C:\\Windows\\Temp')) }}\\python-{{ llama_server_windows_python_version }}-amd64.exe" + arguments: /quiet InstallAllUsers=1 PrependPath=0 Include_test=0 + state: present + when: not __llama_python_check.stat.exists + +- name: Check Microsoft VC++ runtime availability + ansible.windows.win_stat: + path: "C:\\Windows\\System32\\vcruntime140.dll" + register: __llama_vcredist_check + +- name: Download Microsoft VC++ redistributable + ansible.windows.win_get_url: + url: "{{ llama_server_windows_vcredist_url }}" + dest: "{{ (ansible_env.TEMP | default('C:\\Windows\\Temp')) }}\\vc_redist.x64.exe" + force: false + when: not __llama_vcredist_check.stat.exists + +- name: Install Microsoft VC++ redistributable + ansible.windows.win_package: + path: "{{ (ansible_env.TEMP | default('C:\\Windows\\Temp')) }}\\vc_redist.x64.exe" + arguments: /install /quiet /norestart + state: present + when: not __llama_vcredist_check.stat.exists + +- name: Download llama.cpp Windows binary + ansible.windows.win_get_url: + url: "{{ __llama_release_zip_url }}" + dest: "{{ __llama_release_zip_path }}" + force: false + +- name: Check deployed llama.cpp release marker + ansible.windows.win_stat: + path: "{{ __llama_release_marker }}" + register: __llama_release_marker_stat + +- name: Remove stale llama-server binary before release upgrade + ansible.windows.win_file: + path: "{{ llama_server_windows_install_dir }}\\{{ llama_server_windows_bin_name }}" + state: absent + when: not __llama_release_marker_stat.stat.exists + +- name: Extract llama.cpp binary + community.windows.win_unzip: + src: "{{ __llama_release_zip_path }}" + dest: "{{ llama_server_windows_install_dir }}" + delete_archive: false + when: not __llama_release_marker_stat.stat.exists + +- name: Mark deployed llama.cpp release + ansible.windows.win_copy: + content: "{{ __llama_release_asset_name }}" + dest: "{{ __llama_release_marker }}" + when: not __llama_release_marker_stat.stat.exists + +- name: Verify llama-server binary is present + ansible.windows.win_stat: + path: "{{ llama_server_windows_install_dir }}\\{{ llama_server_windows_bin_name }}" + register: __llama_bin_stat + +- name: Fail if llama-server binary is missing + ansible.builtin.fail: + msg: >- + llama-server binary not found at {{ llama_server_windows_install_dir }}\{{ llama_server_windows_bin_name }}. + Check download/unzip and llama_server_windows_backend settings. + when: not __llama_bin_stat.stat.exists + +- name: Check if NSSM is installed + ansible.windows.win_stat: + path: "{{ llama_server_windows_nssm_dir }}\\nssm-{{ llama_server_windows_nssm_version }}\\win64\\nssm.exe" + register: __nssm_stat + +- name: Download NSSM if not installed + ansible.windows.win_get_url: + url: "{{ llama_server_windows_nssm_url }}" + dest: "{{ __nssm_zip_path }}" + force: false + when: not __nssm_stat.stat.exists + +- name: Extract NSSM if not installed + community.windows.win_unzip: + src: "{{ __nssm_zip_path }}" + dest: "{{ llama_server_windows_nssm_dir }}" + delete_archive: false + creates: "{{ llama_server_windows_nssm_dir }}\\nssm-{{ llama_server_windows_nssm_version }}\\win64\\nssm.exe" + when: not __nssm_stat.stat.exists + +- name: Copy NSSM executable + ansible.windows.win_copy: + src: "{{ llama_server_windows_nssm_dir }}\\nssm-{{ llama_server_windows_nssm_version }}\\win64\\nssm.exe" + dest: "{{ __nssm_exe }}" + remote_src: true + +- name: Copy Windows-compatible Hugging Face sync script + ansible.windows.win_copy: + src: scripts/llama_hf_sync_windows.py + dest: "{{ __llama_sync_script }}" + mode: "0750" + +- name: Write desired llama server model manifest input + ansible.windows.win_copy: + content: "{{ llama_server_models | to_nice_json }}" + dest: "{{ __llama_models_json }}" + +- name: Write global llama server preset options + ansible.windows.win_copy: + content: "{{ llama_server_preset_global | to_nice_json }}" + dest: "{{ __llama_preset_global_json }}" + +- name: Check llama sync virtualenv + ansible.windows.win_stat: + path: "{{ __llama_venv_python }}" + register: __llama_venv_python_check + +- name: Create llama sync virtualenv + ansible.windows.win_command: + argv: + - "{{ llama_server_windows_python_path }}" + - -m + - venv + - "{{ __llama_venv_path }}" + changed_when: false + when: not __llama_venv_python_check.stat.exists + +- name: Upgrade pip tooling in llama sync virtualenv + ansible.windows.win_command: + argv: + - "{{ __llama_venv_python }}" + - -m + - pip + - install + - --upgrade + - pip + - setuptools + - packaging + changed_when: false + +- name: Install Hugging Face dependency in llama sync virtualenv + ansible.windows.win_command: + argv: + - "{{ __llama_venv_python }}" + - -m + - pip + - install + - "huggingface_hub>=0.32.0" + changed_when: false + +- name: Run llama server model sync from Hugging Face + ansible.windows.win_command: + cmd: >- + "{{ __llama_venv_python }}" "{{ __llama_sync_script }}" + --models-file "{{ __llama_models_json }}" + --managed-dir "{{ __llama_managed_dir }}" + --links-dir "{{ __llama_links_dir }}" + --manifest-file "{{ __llama_manifest }}" + --preset-file "{{ __llama_preset_file }}" + --preset-global-file "{{ __llama_preset_global_json }}" + --container-links-dir "{{ __llama_links_dir | replace('\\', '/') }}" + {% if llama_server_models_prune_enabled | default(true) %}--prune{% endif %} + {% if llama_server_models_dry_run | default(false) %}--dry-run{% endif %} + register: __llama_sync_output__ + changed_when: false + environment: + HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') | default('', true) }}" + +- name: Parse llama server sync output + ansible.builtin.set_fact: + __llama_sync_result__: "{{ __llama_sync_output__.stdout | from_json }}" + +- name: Stat llama router preset file (when not in dry-run mode) + ansible.windows.win_stat: + path: "{{ __llama_preset_file }}" + register: __llama_preset_stat__ + when: not (llama_server_models_dry_run | default(false)) + +- name: Ensure router preset file exists when not in dry-run mode + ansible.builtin.assert: + that: + - __llama_preset_stat__.stat.exists + fail_msg: "llama router preset file was not created" + when: not (llama_server_models_dry_run | default(false)) + +- name: Remove existing llama-server service if present + ansible.windows.win_service: + name: llama-server + state: absent + failed_when: false + +- name: Install llama-server service via NSSM + ansible.windows.win_command: + argv: >- + {{ + [ + __nssm_exe, + 'install', + 'llama-server', + (llama_server_windows_install_dir ~ '\\' ~ llama_server_windows_bin_name), + '--port', + (llama_server_port | default(11434) | string), + '--host', + (llama_server_host | default('127.0.0.1') | string), + '--metrics', + '--models-preset', + (__llama_preset_file | string), + '--models-max', + (llama_server_models_max | string) + ] + + (llama_server_argv_extra | default([])) + }} + args: + chdir: "{{ llama_server_windows_install_dir }}" + register: __llama_nssm_install + changed_when: __llama_nssm_install.rc == 0 + +- name: Set llama-server AppDirectory via NSSM + ansible.windows.win_command: + argv: + - "{{ __nssm_exe }}" + - set + - llama-server + - AppDirectory + - "{{ llama_server_windows_install_dir }}" + +- name: Configure llama-server stdout log via NSSM + ansible.windows.win_command: + argv: + - "{{ __nssm_exe }}" + - set + - llama-server + - AppStdout + - "{{ llama_server_windows_install_dir }}\\llama-server.out.log" + +- name: Configure llama-server stderr log via NSSM + ansible.windows.win_command: + argv: + - "{{ __nssm_exe }}" + - set + - llama-server + - AppStderr + - "{{ llama_server_windows_install_dir }}\\llama-server.err.log" + +- name: Set llama-server Start mode to AUTO via NSSM + ansible.windows.win_command: + argv: + - "{{ __nssm_exe }}" + - set + - llama-server + - Start + - SERVICE_AUTO_START + +- name: Ensure llama-server service is running + ansible.windows.win_service: + name: llama-server + start_mode: auto + state: restarted + when: not ansible_check_mode | default(false) + +- name: Wait for llama-server to listen on port + ansible.windows.win_wait_for: + port: "{{ llama_server_port | default(11434) }}" + delay: 5 + timeout: 60 + when: not ansible_check_mode | default(false) + +- name: Query llama router model list + ansible.windows.win_uri: + url: "http://127.0.0.1:{{ llama_server_port | default(11434) }}/models?reload=1" + method: GET + status_code: 200 + return_content: true + register: __llama_router_models__ + changed_when: false + when: + - llama_server_models_sync_enabled | default(true) + - not (llama_server_models_dry_run | default(false)) + +- name: Parse llama router model list payload + ansible.builtin.set_fact: + __llama_router_models_payload__: "{{ __llama_router_models__.content | from_json }}" + when: + - llama_server_models_sync_enabled | default(true) + - not (llama_server_models_dry_run | default(false)) + +- name: Validate expected model aliases are available in router + ansible.builtin.assert: + that: + - item.name in (__llama_router_models_payload__.data | map(attribute='id') | list) + fail_msg: "Model name '{{ item.name }}' not present in llama router /models output" + loop: "{{ __llama_sync_result__.models | default([]) }}" + loop_control: + label: "{{ item.name }}" + when: + - llama_server_models_sync_enabled | default(true) + - not (llama_server_models_dry_run | default(false)) + +- name: Set firewall remote IPs when string provided + ansible.builtin.set_fact: + llama_server_windows_remoteip: "{{ llama_server_windows_sources }}" + when: + - llama_server_windows_sources is defined + - llama_server_windows_sources is string + +- name: Set firewall remote IPs when list provided + ansible.builtin.set_fact: + llama_server_windows_remoteip: "{{ llama_server_windows_sources | join(',') }}" + when: + - llama_server_windows_sources is defined + - not (llama_server_windows_sources is string) + +- name: Allow incoming traffic to llama-server port through Windows Firewall + community.windows.win_firewall_rule: + name: Llama Server "{{ llama_server_port | default(11434) }}" + localport: "{{ llama_server_port | default(11434) }}" + protocol: tcp + direction: in + action: allow + enabled: true + # Provide one or more IPs/CIDRs in llama_server_windows_sources (e.g. ['203.0.113.10']) + remoteip: "{{ llama_server_windows_remoteip | default(omit) }}" + state: present + +# --- Windows Exporter tasks --- + +- name: Run Windows Exporter setup + when: llama_server_windows_exporter_enabled | default(true) | bool + ansible.builtin.include_tasks: + file: llama_exporter.yml + tags: + - roles::llama_server::llama_exporter diff --git a/ansible/roles/windows_prometheus/tasks/main.yml b/ansible/roles/windows_prometheus/tasks/main.yml index 88c6f30..b0ae8a5 100644 --- a/ansible/roles/windows_prometheus/tasks/main.yml +++ b/ansible/roles/windows_prometheus/tasks/main.yml @@ -110,7 +110,7 @@ ansible.windows.win_service: name: Prometheus state: absent - ignore_errors: true + failed_when: false - name: Install Prometheus service via NSSM ansible.windows.win_command: >- diff --git a/inventory/hosts b/inventory/hosts index 0cce085..1388fc9 100644 --- a/inventory/hosts +++ b/inventory/hosts @@ -121,6 +121,7 @@ steambox.local [llamacpp] gpu-01.lab.alexpires.me +ci-01.lab.alexpires.me [qwen3_6_host] gpu-01.lab.alexpires.me