feat(vaultwarden): add SMTP configuration options and enhance signup settings
- Introduced SMTP settings for Vaultwarden including host, port, security, and authentication details. - Added variables for signup verification, 2FA settings, password hints, and logging options. - Updated Vaultwarden deployment to utilize new SMTP configurations. - Enhanced Grafana module to support dynamic dashboard and datasource provisioning. - Added LLM proxy configuration for Open Web UI with necessary environment variables.
This commit is contained in:
@@ -0,0 +1,26 @@
|
||||
llama_server_user: "llama"
|
||||
llama_server_group: "llama"
|
||||
llama_server_home: "/home/llama"
|
||||
llama_server_pubkey: ""
|
||||
llama_server_models: []
|
||||
llama_server_models_max: 1
|
||||
llama_server_image: "ghcr.io/ggml-org/llama.cpp"
|
||||
llama_server_tag: "server-vulkan"
|
||||
llama_server_extra_groups: "users,video"
|
||||
llama_server_devices: []
|
||||
llama_server_env: {}
|
||||
llama_server_port: 8080
|
||||
llama_server_preset_global: {}
|
||||
|
||||
# Llama Exporter configuration
|
||||
llama_server_exporter_enabled: true
|
||||
llama_server_exporter_port: 9550
|
||||
llama_server_exporter_user: llama_exporter
|
||||
llama_server_exporter_user_home: "/opt/llama_exporter"
|
||||
llama_server_exporter_install_dir: "/opt/llama_exporter"
|
||||
llama_server_exporter_venv_path: "/opt/llama_exporter/.venv"
|
||||
llama_server_exporter_scrape_interval: 15
|
||||
llama_server_exporter_scrape_timeout: 5
|
||||
|
||||
# Llama.cpp scrape targets
|
||||
llama_server_exporter_targets: []
|
||||
@@ -0,0 +1,389 @@
|
||||
#!/usr/bin/env python3
|
||||
"""LLM inference metrics exporter for Prometheus.
|
||||
|
||||
Background thread scrapes only loaded models to avoid triggering model loads
|
||||
in llama-server, then the HTTP handler serves the cached data immediately.
|
||||
|
||||
Exposes metrics for every known model from /models; unloaded models show
|
||||
zero values without ever requesting /metrics?model=<unloaded>.
|
||||
|
||||
Uses persistent JSON cache on disk to survive restarts and compute counter
|
||||
deltas for Prometheus rate/irate queries.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import threading
|
||||
import logging
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
POLL_INTERVAL = int(os.environ.get("LLAMA_EXPORTER_INTERVAL", "15"))
|
||||
SCRAPE_TIMEOUT = int(os.environ.get("LLAMA_EXPORTER_TIMEOUT", "5"))
|
||||
CACHE_FILE = os.environ.get(
|
||||
"LLAMA_EXPORTER_CACHE",
|
||||
os.path.join(os.path.dirname(os.path.abspath(__file__)), "llama_exporter_cache.json"),
|
||||
)
|
||||
|
||||
LLAMA_CPP_DEFAULTS = [
|
||||
{"name": "llama.cpp", "url": "http://localhost:11434"},
|
||||
]
|
||||
|
||||
# All llama.cpp metrics we expose per model (counters + gauges).
|
||||
ALL_METRICS = [
|
||||
"llamacpp:prompt_tokens_total",
|
||||
"llamacpp:prompt_seconds_total",
|
||||
"llamacpp:tokens_predicted_total",
|
||||
"llamacpp:tokens_predicted_seconds_total",
|
||||
"llamacpp:n_decode_total",
|
||||
"llamacpp:n_tokens_max",
|
||||
"llamacpp:prompt_tokens_seconds",
|
||||
"llamacpp:predicted_tokens_seconds",
|
||||
"llamacpp:requests_processing",
|
||||
"llamacpp:requests_deferred",
|
||||
"llamacpp:n_busy_slots_per_decode",
|
||||
]
|
||||
|
||||
# Counter metrics that accumulate over time — exposed as _delta for rate() queries.
|
||||
COUNTER_METRICS = {
|
||||
"llamacpp:prompt_tokens_total",
|
||||
"llamacpp:prompt_seconds_total",
|
||||
"llamacpp:tokens_predicted_total",
|
||||
"llamacpp:tokens_predicted_seconds_total",
|
||||
"llamacpp:n_decode_total",
|
||||
}
|
||||
|
||||
|
||||
class Cache:
|
||||
"""Persistent, thread-safe metrics cache with delta computation."""
|
||||
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self._data = {} # { model_id: { metric_name: value } }
|
||||
self._known_models = set()
|
||||
self._health = None
|
||||
self._loaded = {}
|
||||
|
||||
# Previous state for delta computation: { model_id: { metric_name: value } }
|
||||
self._prev = {}
|
||||
self._prev_timestamp = 0.0
|
||||
self._deltas = {}
|
||||
|
||||
# Load persisted state
|
||||
self._load_cache()
|
||||
|
||||
def _load_cache(self):
|
||||
"""Load previous scrape state from disk."""
|
||||
try:
|
||||
with open(CACHE_FILE, "r") as f:
|
||||
state = json.load(f)
|
||||
self._prev = state.get("models", {})
|
||||
self._prev_timestamp = state.get("timestamp", 0.0)
|
||||
logger.info("Loaded cache from %s (timestamp=%s, models=%d)",
|
||||
CACHE_FILE, self._prev_timestamp, len(self._prev))
|
||||
except FileNotFoundError:
|
||||
logger.info("No cache file found at %s", CACHE_FILE)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load cache: %s", e)
|
||||
|
||||
def _save_cache(self, current_data, known_models, loaded):
|
||||
"""Save current scrape state to disk."""
|
||||
try:
|
||||
state = {
|
||||
"timestamp": time.time(),
|
||||
"known": list(known_models),
|
||||
"loaded": list(loaded),
|
||||
"models": {},
|
||||
}
|
||||
for mid, metrics in current_data.items():
|
||||
state["models"][mid] = dict(metrics)
|
||||
tmp = CACHE_FILE + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(state, f)
|
||||
os.replace(tmp, CACHE_FILE)
|
||||
except Exception as e:
|
||||
logger.error("Failed to save cache: %s", e)
|
||||
|
||||
def _compute_deltas_for_data(self, current_data):
|
||||
"""Compute deltas given current data and stored previous state.
|
||||
|
||||
Only counter metrics get deltas. On counter reset (value went backward),
|
||||
delta is silently 0 (no entry added).
|
||||
"""
|
||||
deltas = {}
|
||||
if not self._prev:
|
||||
return deltas
|
||||
|
||||
for mid, prev_metrics in self._prev.items():
|
||||
mid_deltas = {}
|
||||
cur_metrics = current_data.get(mid, {})
|
||||
for mname, prev_val in prev_metrics.items():
|
||||
# Only compute deltas for counter metrics
|
||||
if mname not in COUNTER_METRICS:
|
||||
continue
|
||||
cur_val = cur_metrics.get(mname)
|
||||
if cur_val is None:
|
||||
continue
|
||||
diff = cur_val - prev_val
|
||||
# Counter reset: value went backward, delta is 0 (wrapped)
|
||||
if diff < 0:
|
||||
continue
|
||||
if diff > 0:
|
||||
mid_deltas[mname] = diff
|
||||
if mid_deltas:
|
||||
deltas[mid] = mid_deltas
|
||||
return deltas
|
||||
|
||||
def update(self, models_data, metric_lines, known_models=None):
|
||||
"""Called by the background thread after a successful scrape cycle."""
|
||||
with self._lock:
|
||||
# Discover model IDs and loaded status from /models endpoint
|
||||
known = set()
|
||||
loaded = {}
|
||||
if models_data and isinstance(models_data, dict):
|
||||
model_list = models_data.get("data", [])
|
||||
for m in model_list:
|
||||
if not isinstance(m, dict):
|
||||
continue
|
||||
mid = m.get("id", "unknown")
|
||||
known.add(mid)
|
||||
status_data = m.get("status", {})
|
||||
if isinstance(status_data, dict) and status_data.get("value") == "loaded":
|
||||
loaded[mid] = 1.0
|
||||
self._health = {"status": "ok", "model": mid}
|
||||
|
||||
if not known:
|
||||
self._health = {"status": "error", "model": "unknown"}
|
||||
|
||||
# Build new metrics dict for all known models
|
||||
new_data = {}
|
||||
for mid in known:
|
||||
new_data[mid] = {}
|
||||
|
||||
# Apply parsed metric values from /metrics?model=<id>
|
||||
for metric_name, model_id, value in metric_lines:
|
||||
if model_id in known:
|
||||
new_data[model_id][metric_name] = value
|
||||
|
||||
# Ensure every known model has all ALL_METRICS entries
|
||||
for mid in known:
|
||||
for mname in ALL_METRICS:
|
||||
if mname not in new_data[mid]:
|
||||
new_data[mid][mname] = 0.0
|
||||
|
||||
# Previously known models no longer in the list get zeroed out
|
||||
for mid in self._known_models - known:
|
||||
new_data[mid] = {m: 0.0 for m in ALL_METRICS}
|
||||
|
||||
# Compute deltas before updating previous state
|
||||
deltas = self._compute_deltas_for_data(new_data)
|
||||
|
||||
# Save previous state for next cycle
|
||||
self._save_cache(new_data, known, loaded)
|
||||
|
||||
# Update state
|
||||
self._prev = {mid: dict(metrics) for mid, metrics in new_data.items()}
|
||||
self._known_models = known
|
||||
self._data = new_data
|
||||
self._loaded = loaded
|
||||
self._deltas = deltas
|
||||
|
||||
def snapshot(self):
|
||||
"""Return a frozen copy of the current cache state including deltas."""
|
||||
with self._lock:
|
||||
return {
|
||||
"data": {k: dict(v) for k, v in self._data.items()},
|
||||
"known": set(self._known_models),
|
||||
"health": dict(self._health) if self._health else {"status": "error", "model": "unknown"},
|
||||
"loaded": dict(self._loaded),
|
||||
"deltas": {k: dict(v) for k, v in self._deltas.items()},
|
||||
}
|
||||
|
||||
|
||||
cache = Cache()
|
||||
|
||||
|
||||
def _fetch_json(url, timeout=SCRAPE_TIMEOUT):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"Accept": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.debug("Failed to fetch %s: %s", url, e)
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_text(url, timeout=SCRAPE_TIMEOUT):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"Accept": "text/plain"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.read().decode("utf-8")
|
||||
except Exception as e:
|
||||
logger.debug("Failed to fetch %s: %s", url, e)
|
||||
return None
|
||||
|
||||
|
||||
def _scrape_cycle():
|
||||
"""One full scrape cycle: discover models, then scrape metrics per model."""
|
||||
targets = []
|
||||
env_targets = os.environ.get("LLAMA_TARGETS", "")
|
||||
if env_targets:
|
||||
try:
|
||||
targets = json.loads(env_targets)
|
||||
except json.JSONDecodeError:
|
||||
targets = []
|
||||
if not targets:
|
||||
targets = LLAMA_CPP_DEFAULTS
|
||||
|
||||
all_metric_lines = []
|
||||
models_data = None
|
||||
known_models = {} # { model_id: base_url, ... }
|
||||
|
||||
for target in targets:
|
||||
base_url = target["url"].rstrip("/")
|
||||
|
||||
# Fetch /models to discover models and their status
|
||||
models_data = _fetch_json(f"{base_url}/models")
|
||||
|
||||
if models_data and isinstance(models_data, dict):
|
||||
model_list = models_data.get("data", [])
|
||||
for m in model_list:
|
||||
if not isinstance(m, dict) or "id" not in m:
|
||||
continue
|
||||
model_id = m["id"]
|
||||
known_models[model_id] = base_url
|
||||
# Only scrape metrics from loaded models to avoid triggering loads
|
||||
status_data = m.get("status", {})
|
||||
if isinstance(status_data, dict) and status_data.get("value") != "loaded":
|
||||
continue
|
||||
# Scrape /metrics for loaded models only
|
||||
metrics_url = f"{base_url}/metrics?model={model_id}"
|
||||
body = _fetch_text(metrics_url)
|
||||
if body:
|
||||
for line in body.splitlines():
|
||||
parsed = _parse_metric_line(line)
|
||||
if parsed:
|
||||
metric_name, metric_value = parsed
|
||||
if metric_name in ALL_METRICS:
|
||||
all_metric_lines.append((metric_name, model_id, metric_value))
|
||||
else:
|
||||
logger.debug("No metrics body for model %s", model_id)
|
||||
|
||||
# Update the shared cache
|
||||
cache.update(models_data, all_metric_lines, known_models)
|
||||
|
||||
|
||||
def _parse_metric_line(line):
|
||||
"""Parse a single Prometheus metric line. Returns (name, value) or None."""
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
return None
|
||||
try:
|
||||
# Handle lines with labels: metric_name{label="val"} value
|
||||
if "{" in line:
|
||||
parts = line.split("{")
|
||||
name = parts[0].strip()
|
||||
rest = parts[1]
|
||||
# value is the last token after the closing }
|
||||
value = rest.rsplit("}", 1)[-1].strip().split()[-1] if "}" in rest else rest.strip()
|
||||
else:
|
||||
parts = line.split()
|
||||
name = parts[0]
|
||||
value = parts[1] if len(parts) >= 2 else "1"
|
||||
# Try to parse as float
|
||||
float(value)
|
||||
return (name, float(value))
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def _background_scrape():
|
||||
"""Background thread: periodically scrape and update cache."""
|
||||
logger.info("Background scraper started (interval=%ds)", POLL_INTERVAL)
|
||||
# Do one immediate scrape on startup
|
||||
_scrape_cycle()
|
||||
while True:
|
||||
try:
|
||||
time.sleep(POLL_INTERVAL)
|
||||
_scrape_cycle()
|
||||
except Exception as e:
|
||||
logger.error("Scrape cycle error: %s", e)
|
||||
|
||||
|
||||
class MetricsHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
if self.path != "/metrics":
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
snap = cache.snapshot()
|
||||
lines = []
|
||||
|
||||
def _fmt(metric_name, model_id, value):
|
||||
return metric_name + '{' + 'server="llama-cpp-11434",model="' + model_id + '"} ' + str(value)
|
||||
|
||||
# Health metric
|
||||
health = snap["health"]
|
||||
status = health.get("status", "error")
|
||||
health_model = health.get("model", "unknown")
|
||||
health_val = 1.0 if status == "ok" else 0.0
|
||||
lines.append(_fmt("llama_server_health", health_model, health_val))
|
||||
|
||||
# Loaded metrics
|
||||
for mid in snap["loaded"]:
|
||||
lines.append(_fmt("llama_models_loaded", mid, snap["loaded"][mid]))
|
||||
|
||||
# Per-model metrics from cache (absolute values)
|
||||
for mid in sorted(snap["data"]):
|
||||
metrics = snap["data"][mid]
|
||||
for mname in ALL_METRICS:
|
||||
value = metrics.get(mname, 0.0)
|
||||
lines.append(_fmt(mname, mid, value))
|
||||
|
||||
# Per-model delta metrics (counters as change since last scrape)
|
||||
deltas = snap.get("deltas", {})
|
||||
for mid in sorted(deltas):
|
||||
delta_metrics = deltas[mid]
|
||||
for mname in sorted(delta_metrics):
|
||||
value = delta_metrics[mname]
|
||||
lines.append(_fmt(mname + "_delta", mid, value))
|
||||
|
||||
body = "\n".join(lines) + "\n" if lines else "# no metrics available\n"
|
||||
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
self.end_headers()
|
||||
self.wfile.write(body.encode("utf-8"))
|
||||
|
||||
def log_message(self, format, *args):
|
||||
logger.debug("%s - - %s", self.address_string(), format % args)
|
||||
|
||||
|
||||
def main():
|
||||
port = int(os.environ.get("LLAMA_EXPORTER_PORT", "9550"))
|
||||
host = os.environ.get("LLAMA_EXPORTER_BIND", "0.0.0.0")
|
||||
|
||||
# Start background scraper thread
|
||||
scraper = threading.Thread(target=_background_scrape, daemon=True)
|
||||
scraper.start()
|
||||
|
||||
# Start HTTP server
|
||||
server = HTTPServer((host, port), MetricsHandler)
|
||||
logger.info("Starting Llama Exporter on %s:%d (interval=%ds, timeout=%ds, cache=%s)",
|
||||
host, port, POLL_INTERVAL, SCRAPE_TIMEOUT, CACHE_FILE)
|
||||
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Shutting down")
|
||||
server.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Synchronize llama.cpp models from Hugging Face into a managed local directory."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import configparser
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from huggingface_hub import HfApi, snapshot_download
|
||||
|
||||
|
||||
def _sanitize_slug(value: str) -> str:
|
||||
slug = re.sub(r"[^a-zA-Z0-9._-]+", "-", value.strip())
|
||||
slug = slug.strip("-._")
|
||||
return slug.lower() or "model"
|
||||
|
||||
|
||||
def _resolve_repo_file(api: HfApi, model: dict[str, Any]) -> str:
|
||||
repo_files = api.list_repo_files(repo_id=model["model_id"], revision=model["revision"], repo_type="model")
|
||||
|
||||
hf_file = model.get("hf_file")
|
||||
if hf_file:
|
||||
if hf_file in repo_files:
|
||||
return hf_file
|
||||
raise RuntimeError(
|
||||
f"Requested hf_file '{hf_file}' was not found in repo "
|
||||
f"{model['model_id']}@{model['revision']}"
|
||||
)
|
||||
|
||||
quant_upper = (model.get("quant") or "").upper()
|
||||
ggufs = [item for item in repo_files if item.lower().endswith(".gguf")]
|
||||
matches = [item for item in ggufs if quant_upper in Path(item).name.upper()]
|
||||
if matches:
|
||||
return sorted(matches, key=lambda name: (len(Path(name).name), name))[0]
|
||||
|
||||
raise RuntimeError(
|
||||
f"No GGUF file containing quant '{model.get('quant')}' found in "
|
||||
f"repo {model['model_id']}@{model['revision']}. Set hf_file explicitly if needed."
|
||||
)
|
||||
|
||||
|
||||
def _load_models(models_file: Path) -> list[dict[str, Any]]:
|
||||
data = json.loads(models_file.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, list):
|
||||
raise RuntimeError("models file must contain a JSON array")
|
||||
|
||||
normalized: list[dict[str, Any]] = []
|
||||
for item in data:
|
||||
if not isinstance(item, dict):
|
||||
raise RuntimeError("each model entry must be an object")
|
||||
if item.get("enable", True) is False:
|
||||
continue
|
||||
|
||||
model_id = str(item.get("model_id", "")).strip()
|
||||
quant = str(item.get("quant", "")).strip()
|
||||
revision = str(item.get("revision", "")).strip()
|
||||
hf_file = str(item.get("hf_file", "")).strip() or None
|
||||
name = str(item.get("name", "")).strip()
|
||||
preset = item.get("preset", {})
|
||||
|
||||
if preset is None:
|
||||
preset = {}
|
||||
if not isinstance(preset, dict):
|
||||
raise RuntimeError(f"preset must be an object for model '{name or model_id or 'unknown'}'")
|
||||
|
||||
if not name:
|
||||
raise RuntimeError(f"name is required for model '{model_id or 'unknown'}'")
|
||||
if not model_id:
|
||||
raise RuntimeError("model_id is required for all models")
|
||||
if not quant and not hf_file:
|
||||
raise RuntimeError(f"quant or hf_file is required for model '{model_id}'")
|
||||
if not revision:
|
||||
raise RuntimeError(f"revision is required for model '{model_id}'")
|
||||
|
||||
normalized.append(
|
||||
{
|
||||
"model_id": model_id,
|
||||
"quant": quant,
|
||||
"revision": revision,
|
||||
"hf_file": hf_file,
|
||||
"name": name,
|
||||
"preset": preset,
|
||||
}
|
||||
)
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def _load_preset_global(preset_global_file: Path | None) -> dict[str, Any]:
|
||||
if preset_global_file is None:
|
||||
return {}
|
||||
|
||||
if not preset_global_file.exists():
|
||||
return {}
|
||||
|
||||
data = json.loads(preset_global_file.read_text(encoding="utf-8"))
|
||||
if data is None:
|
||||
return {}
|
||||
if not isinstance(data, dict):
|
||||
raise RuntimeError("global preset options must be a JSON object")
|
||||
return data
|
||||
|
||||
|
||||
def _to_preset_value(value: Any) -> str:
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, (int, float, str)):
|
||||
return str(value)
|
||||
return json.dumps(value, separators=(",", ":"))
|
||||
|
||||
|
||||
def _normalize_preset_options(options: dict[str, Any], scope: str) -> dict[str, str]:
|
||||
normalized: dict[str, str] = {}
|
||||
for key, value in options.items():
|
||||
key_str = str(key).strip()
|
||||
if not key_str:
|
||||
raise RuntimeError(f"{scope} preset option keys must be non-empty")
|
||||
if value is None:
|
||||
continue
|
||||
normalized[key_str] = _to_preset_value(value)
|
||||
return normalized
|
||||
|
||||
|
||||
def _download_model(model: dict[str, Any], target_dir: Path, repo_file: str, dry_run: bool) -> None:
|
||||
if dry_run:
|
||||
return
|
||||
|
||||
snapshot_download(
|
||||
repo_id=model["model_id"],
|
||||
revision=model["revision"],
|
||||
local_dir=str(target_dir),
|
||||
allow_patterns=[repo_file],
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
|
||||
def _write_preset(preset_file: Path, entries: list[dict[str, Any]], global_options: dict[str, Any]) -> None:
|
||||
parser = configparser.ConfigParser(interpolation=None)
|
||||
parser.optionxform = str
|
||||
parser["*"] = _normalize_preset_options(global_options, "global")
|
||||
|
||||
for entry in entries:
|
||||
model_options = _normalize_preset_options(entry.get("preset", {}), f"model '{entry['name']}'")
|
||||
model_options["model"] = entry["container_model_path"]
|
||||
parser[entry["name"]] = model_options
|
||||
|
||||
preset_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with preset_file.open("w", encoding="utf-8") as fh:
|
||||
# Router preset format supports top-level version key.
|
||||
fh.write("version = 1\n\n")
|
||||
parser.write(fh)
|
||||
|
||||
|
||||
def _prune_unmanaged(managed_dir: Path, links_dir: Path, keep_slugs: set[str], dry_run: bool) -> tuple[list[str], list[str]]:
|
||||
pruned_dirs: list[str] = []
|
||||
pruned_links: list[str] = []
|
||||
|
||||
if managed_dir.exists():
|
||||
for child in managed_dir.iterdir():
|
||||
if not child.is_dir():
|
||||
continue
|
||||
if child.name == ".router":
|
||||
continue
|
||||
if child.name not in keep_slugs:
|
||||
pruned_dirs.append(child.name)
|
||||
if not dry_run:
|
||||
shutil.rmtree(child)
|
||||
|
||||
if links_dir.exists():
|
||||
for child in links_dir.iterdir():
|
||||
if child.suffix != ".gguf":
|
||||
continue
|
||||
slug = child.stem
|
||||
if slug not in keep_slugs:
|
||||
pruned_links.append(child.name)
|
||||
if not dry_run:
|
||||
child.unlink(missing_ok=True)
|
||||
|
||||
return pruned_dirs, pruned_links
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Sync Hugging Face GGUF models for llama.cpp router mode")
|
||||
parser.add_argument("--models-file", required=True)
|
||||
parser.add_argument("--managed-dir", required=True)
|
||||
parser.add_argument("--links-dir", required=True)
|
||||
parser.add_argument("--manifest-file", required=True)
|
||||
parser.add_argument("--preset-file", required=True)
|
||||
parser.add_argument("--preset-global-file")
|
||||
parser.add_argument("--container-links-dir", required=True)
|
||||
parser.add_argument("--prune", action="store_true")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
models_file = Path(args.models_file)
|
||||
managed_dir = Path(args.managed_dir)
|
||||
links_dir = Path(args.links_dir)
|
||||
manifest_file = Path(args.manifest_file)
|
||||
preset_file = Path(args.preset_file)
|
||||
preset_global_file = Path(args.preset_global_file) if args.preset_global_file else None
|
||||
|
||||
managed_dir.mkdir(parents=True, exist_ok=True)
|
||||
links_dir.mkdir(parents=True, exist_ok=True)
|
||||
manifest_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
preset_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
models = _load_models(models_file)
|
||||
global_options = _load_preset_global(preset_global_file)
|
||||
api = HfApi()
|
||||
entries: list[dict[str, Any]] = []
|
||||
|
||||
for model in models:
|
||||
slug = _sanitize_slug(model["name"])
|
||||
target_dir = managed_dir / slug
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
repo_file = _resolve_repo_file(api, model)
|
||||
|
||||
_download_model(model, target_dir, repo_file, args.dry_run)
|
||||
selected_file = target_dir / repo_file
|
||||
if not args.dry_run and not selected_file.exists():
|
||||
raise RuntimeError(f"Expected downloaded file not found: {selected_file}")
|
||||
|
||||
link_path = links_dir / f"{slug}.gguf"
|
||||
container_model_path = f"{args.container_links_dir.rstrip('/')}/{slug}.gguf"
|
||||
|
||||
if not args.dry_run:
|
||||
if link_path.exists() or link_path.is_symlink():
|
||||
link_path.unlink()
|
||||
# Keep symlink targets relative so they remain valid inside the
|
||||
# container-mounted /models tree.
|
||||
relative_target = os.path.relpath(selected_file, start=link_path.parent)
|
||||
link_path.symlink_to(relative_target)
|
||||
|
||||
entries.append(
|
||||
{
|
||||
"name": model["name"],
|
||||
"slug": slug,
|
||||
"model_id": model["model_id"],
|
||||
"revision": model["revision"],
|
||||
"quant": model["quant"],
|
||||
"hf_file": model.get("hf_file"),
|
||||
"preset": model.get("preset", {}),
|
||||
"repo_file": repo_file,
|
||||
"selected_file": str(selected_file),
|
||||
"link_path": str(link_path),
|
||||
"container_model_path": container_model_path,
|
||||
}
|
||||
)
|
||||
|
||||
pruned_dirs: list[str] = []
|
||||
pruned_links: list[str] = []
|
||||
if args.prune:
|
||||
keep_slugs = {entry["slug"] for entry in entries}
|
||||
pruned_dirs, pruned_links = _prune_unmanaged(managed_dir, links_dir, keep_slugs, args.dry_run)
|
||||
|
||||
result = {
|
||||
"models": entries,
|
||||
"dry_run": args.dry_run,
|
||||
"prune": args.prune,
|
||||
"pruned_dirs": pruned_dirs,
|
||||
"pruned_links": pruned_links,
|
||||
}
|
||||
|
||||
if not args.dry_run:
|
||||
manifest_file.write_text(json.dumps(result, indent=2), encoding="utf-8")
|
||||
_write_preset(preset_file, entries, global_options)
|
||||
|
||||
print(json.dumps(result))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except Exception as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for llama_exporter cache persistence and delta computation."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import shutil
|
||||
import unittest
|
||||
|
||||
# Setup temp cache file before importing the module
|
||||
TEMP_DIR = tempfile.mkdtemp()
|
||||
os.environ["LLAMA_EXPORTER_CACHE"] = os.path.join(TEMP_DIR, "test_cache.json")
|
||||
|
||||
# Import from parent directory (scripts)
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from llama_exporter import Cache
|
||||
|
||||
|
||||
class TestCachePersistenceAndDeltas(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
# Ensure temp dir exists (pytest may clean it between test methods)
|
||||
os.makedirs(TEMP_DIR, exist_ok=True)
|
||||
self.cache = Cache()
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(TEMP_DIR, ignore_errors=True)
|
||||
|
||||
def test_cold_start_no_deltas(self):
|
||||
"""First scrape should produce no deltas since there's no previous state."""
|
||||
models_data = {
|
||||
"data": [
|
||||
{"id": "model-v1", "status": {"value": "loaded"}},
|
||||
{"id": "model-v2", "status": {"value": "unloaded"}},
|
||||
]
|
||||
}
|
||||
metric_lines = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 100.0),
|
||||
("llamacpp:prompt_seconds_total", "model-v1", 2.5),
|
||||
("llamacpp:n_tokens_max", "model-v1", 4096.0),
|
||||
]
|
||||
known_models = {"model-v1": "http://localhost:11434", "model-v2": "http://localhost:11434"}
|
||||
|
||||
self.cache.update(models_data, metric_lines, known_models)
|
||||
|
||||
snap = self.cache.snapshot()
|
||||
# No deltas on first scrape
|
||||
self.assertEqual(snap["deltas"], {})
|
||||
# Absolute values are correct
|
||||
self.assertEqual(snap["data"]["model-v1"]["llamacpp:prompt_tokens_total"], 100.0)
|
||||
# Unloaded model has zeros
|
||||
self.assertEqual(snap["data"]["model-v2"]["llamacpp:prompt_tokens_total"], 0.0)
|
||||
# Only loaded model is in loaded set
|
||||
self.assertIn("model-v1", snap["loaded"])
|
||||
self.assertNotIn("model-v2", snap["loaded"])
|
||||
|
||||
def test_delta_computation(self):
|
||||
"""Second scrape should produce deltas from first scrape."""
|
||||
models_data = {
|
||||
"data": [
|
||||
{"id": "model-v1", "status": {"value": "loaded"}},
|
||||
]
|
||||
}
|
||||
|
||||
# First scrape
|
||||
metric_lines_1 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 100.0),
|
||||
("llamacpp:prompt_seconds_total", "model-v1", 2.5),
|
||||
]
|
||||
known_models_1 = {"model-v1": "http://localhost:11434"}
|
||||
self.cache.update(models_data, metric_lines_1, known_models_1)
|
||||
|
||||
# Second scrape - values increased
|
||||
metric_lines_2 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 250.0),
|
||||
("llamacpp:prompt_seconds_total", "model-v1", 5.0),
|
||||
]
|
||||
self.cache.update(models_data, metric_lines_2, known_models_1)
|
||||
|
||||
snap = self.cache.snapshot()
|
||||
# Deltas should be computed
|
||||
self.assertIn("model-v1", snap["deltas"])
|
||||
self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_tokens_total"], 150.0)
|
||||
self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_seconds_total"], 2.5)
|
||||
|
||||
def test_counter_reset(self):
|
||||
"""If a counter value goes backward, delta should be 0."""
|
||||
models_data = {
|
||||
"data": [
|
||||
{"id": "model-v1", "status": {"value": "loaded"}},
|
||||
]
|
||||
}
|
||||
|
||||
# First scrape
|
||||
metric_lines_1 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 100.0),
|
||||
]
|
||||
known_models_1 = {"model-v1": "http://localhost:11434"}
|
||||
self.cache.update(models_data, metric_lines_1, known_models_1)
|
||||
|
||||
# Second scrape - counter reset (model was reloaded)
|
||||
metric_lines_2 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 10.0),
|
||||
]
|
||||
self.cache.update(models_data, metric_lines_2, known_models_1)
|
||||
|
||||
snap = self.cache.snapshot()
|
||||
# On counter reset, delta is implicitly 0 (no entry in deltas)
|
||||
self.assertNotIn("model-v1", snap["deltas"])
|
||||
|
||||
def test_gauge_metrics_no_delta(self):
|
||||
"""Gauge metrics should not get delta entries (only counters do)."""
|
||||
models_data = {
|
||||
"data": [
|
||||
{"id": "model-v1", "status": {"value": "loaded"}},
|
||||
]
|
||||
}
|
||||
|
||||
# First scrape
|
||||
metric_lines_1 = [
|
||||
("llamacpp:n_tokens_max", "model-v1", 4096.0),
|
||||
("llamacpp:requests_processing", "model-v1", 2.0),
|
||||
]
|
||||
known_models_1 = {"model-v1": "http://localhost:11434"}
|
||||
self.cache.update(models_data, metric_lines_1, known_models_1)
|
||||
|
||||
# Second scrape
|
||||
metric_lines_2 = [
|
||||
("llamacpp:n_tokens_max", "model-v1", 8192.0),
|
||||
("llamacpp:requests_processing", "model-v1", 5.0),
|
||||
]
|
||||
self.cache.update(models_data, metric_lines_2, known_models_1)
|
||||
|
||||
snap = self.cache.snapshot()
|
||||
# Gauge metrics should not have deltas
|
||||
self.assertNotIn("model-v1", snap["deltas"])
|
||||
|
||||
def test_persistence_across_restarts(self):
|
||||
"""Delta computation should work across cache reloads from disk."""
|
||||
models_data = {
|
||||
"data": [
|
||||
{"id": "model-v1", "status": {"value": "loaded"}},
|
||||
]
|
||||
}
|
||||
|
||||
# First scrape (first process instance)
|
||||
metric_lines_1 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 100.0),
|
||||
("llamacpp:prompt_seconds_total", "model-v1", 2.5),
|
||||
]
|
||||
known_models_1 = {"model-v1": "http://localhost:11434"}
|
||||
self.cache.update(models_data, metric_lines_1, known_models_1)
|
||||
|
||||
# Simulate restart: create new Cache instance (loads from disk)
|
||||
self.cache._save_cache(self.cache._data, self.cache._known_models, self.cache._loaded)
|
||||
|
||||
# Create a new cache instance (simulates restart)
|
||||
new_cache = Cache()
|
||||
|
||||
# Second scrape after restart
|
||||
metric_lines_2 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 300.0),
|
||||
("llamacpp:prompt_seconds_total", "model-v1", 8.0),
|
||||
]
|
||||
known_models_2 = {"model-v1": "http://localhost:11434"}
|
||||
new_cache.update(models_data, metric_lines_2, known_models_2)
|
||||
|
||||
snap = new_cache.snapshot()
|
||||
# Deltas should be computed using persisted previous state
|
||||
self.assertIn("model-v1", snap["deltas"])
|
||||
self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_tokens_total"], 200.0)
|
||||
self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_seconds_total"], 5.5)
|
||||
|
||||
def test_unloaded_model_zero_values(self):
|
||||
"""Unloaded models should appear with zero values but never trigger /metrics calls."""
|
||||
models_data = {
|
||||
"data": [
|
||||
{"id": "model-v1", "status": {"value": "loaded"}},
|
||||
{"id": "model-v2", "status": {"value": "unloaded"}},
|
||||
{"id": "model-v3", "status": {"value": "unloaded"}},
|
||||
]
|
||||
}
|
||||
metric_lines = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 50.0),
|
||||
]
|
||||
known_models = {
|
||||
"model-v1": "http://localhost:11434",
|
||||
"model-v2": "http://localhost:11434",
|
||||
"model-v3": "http://localhost:11434",
|
||||
}
|
||||
|
||||
self.cache.update(models_data, metric_lines, known_models)
|
||||
|
||||
snap = self.cache.snapshot()
|
||||
# Loaded model has its actual value
|
||||
self.assertEqual(snap["data"]["model-v1"]["llamacpp:prompt_tokens_total"], 50.0)
|
||||
# Unloaded models have zeros for all metrics
|
||||
for m in ["model-v2", "model-v3"]:
|
||||
self.assertEqual(snap["data"][m]["llamacpp:prompt_tokens_total"], 0.0)
|
||||
self.assertEqual(snap["data"][m]["llamacpp:n_tokens_max"], 0.0)
|
||||
|
||||
def test_removed_model_zeroed(self):
|
||||
"""Models removed from llama-server should be zeroed out."""
|
||||
models_data = {
|
||||
"data": [
|
||||
{"id": "model-v1", "status": {"value": "loaded"}},
|
||||
]
|
||||
}
|
||||
|
||||
# First scrape with two models
|
||||
metric_lines_1 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 100.0),
|
||||
]
|
||||
known_models_1 = {"model-v1": "http://localhost:11434"}
|
||||
self.cache.update(models_data, metric_lines_1, known_models_1)
|
||||
|
||||
# Second scrape with only model-v1 (model-v2 was removed)
|
||||
metric_lines_2 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 200.0),
|
||||
]
|
||||
known_models_2 = {"model-v1": "http://localhost:11434"}
|
||||
self.cache.update(models_data, metric_lines_2, known_models_2)
|
||||
|
||||
snap = self.cache.snapshot()
|
||||
# model-v1 delta should be 100
|
||||
self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_tokens_total"], 100.0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
- name: Reload systemd daemon
|
||||
become: true
|
||||
ansible.builtin.systemd:
|
||||
daemon_reload: true
|
||||
listen: Reload systemd daemon
|
||||
|
||||
- name: Restart llama_exporter
|
||||
become: true
|
||||
ansible.builtin.systemd:
|
||||
name: "{{ llama_server_exporter_service_name }}"
|
||||
state: restarted
|
||||
listen: Restart llama_exporter
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
galaxy_info:
|
||||
author: Alexandre Pires
|
||||
description: Sync llama models from Hugging Face
|
||||
company: A13Labs
|
||||
role_name: llama_models_sync
|
||||
namespace: a13labs
|
||||
license: MIT
|
||||
min_ansible_version: "2.1"
|
||||
platforms:
|
||||
- name: EL
|
||||
versions:
|
||||
- "9"
|
||||
@@ -0,0 +1,73 @@
|
||||
---
|
||||
- name: Create llama_exporter group
|
||||
become: true
|
||||
ansible.builtin.group:
|
||||
name: "{{ llama_server_exporter_user }}"
|
||||
state: present
|
||||
system: true
|
||||
|
||||
- name: Create llama_exporter user
|
||||
become: true
|
||||
ansible.builtin.user:
|
||||
name: "{{ llama_server_exporter_user }}"
|
||||
group: "{{ llama_server_exporter_user }}"
|
||||
home: "{{ llama_server_exporter_user_home }}"
|
||||
create_home: true
|
||||
shell: /usr/sbin/nologin
|
||||
system: true
|
||||
|
||||
- name: Create llama_exporter install directory
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ llama_server_exporter_user }}"
|
||||
group: "{{ llama_server_exporter_user }}"
|
||||
mode: "0755"
|
||||
loop:
|
||||
- "{{ llama_server_exporter_install_dir }}"
|
||||
- "{{ llama_server_exporter_install_dir }}/logs"
|
||||
|
||||
- name: Copy llama_exporter script
|
||||
become: true
|
||||
ansible.builtin.copy:
|
||||
src: scripts/llama_exporter.py
|
||||
dest: "{{ llama_server_exporter_script_path }}"
|
||||
owner: "{{ llama_server_exporter_user }}"
|
||||
group: "{{ llama_server_exporter_user }}"
|
||||
mode: "0755"
|
||||
notify:
|
||||
- Restart llama_exporter
|
||||
|
||||
- name: Create Python virtual environment
|
||||
become: true
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ llama_server_exporter_venv_path }}"
|
||||
creates: "{{ llama_server_exporter_venv_path }}/bin/activate"
|
||||
|
||||
- name: Fix ownership of llama_exporter directory
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "{{ llama_server_exporter_install_dir }}"
|
||||
owner: "{{ llama_server_exporter_user }}"
|
||||
group: "{{ llama_server_exporter_user }}"
|
||||
recurse: true
|
||||
|
||||
- name: Generate llama_exporter systemd service
|
||||
become: true
|
||||
ansible.builtin.template:
|
||||
src: llama_exporter.service.j2
|
||||
dest: "/etc/systemd/system/{{ llama_server_exporter_service_name }}.service"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify:
|
||||
- Reload systemd daemon
|
||||
- Restart llama_exporter
|
||||
|
||||
- name: Enable llama_exporter service
|
||||
become: true
|
||||
ansible.builtin.systemd:
|
||||
name: "{{ llama_server_exporter_service_name }}"
|
||||
enabled: true
|
||||
state: started
|
||||
@@ -0,0 +1,358 @@
|
||||
---
|
||||
- name: Validate llama_server_models schema
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- llama_server_models is iterable
|
||||
fail_msg: "llama_server_models must be a list"
|
||||
|
||||
- name: Create group
|
||||
become: true
|
||||
ansible.builtin.group:
|
||||
name: "{{ llama_server_group }}"
|
||||
system: true
|
||||
|
||||
- name: Create user
|
||||
become: true
|
||||
ansible.builtin.user:
|
||||
name: "{{ llama_server_user }}"
|
||||
groups: "{{ podman_extra_groups | default([]) }}"
|
||||
shell: /bin/bash
|
||||
home: "{{ llama_server_home }}"
|
||||
group: "{{ llama_server_group }}"
|
||||
|
||||
- name: Disable password login for user
|
||||
ansible.builtin.command: passwd -d "{{ llama_server_user }}"
|
||||
become: true
|
||||
changed_when: false
|
||||
|
||||
- name: Create user home
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "{{ llama_server_home }}"
|
||||
state: directory
|
||||
owner: "{{ llama_server_user }}"
|
||||
group: "{{ llama_server_group }}"
|
||||
mode: "0750"
|
||||
|
||||
- name: Set fact podman_binary (Debian family)
|
||||
ansible.builtin.set_fact:
|
||||
podman_binary: /usr/bin/podman
|
||||
when: ansible_os_family == 'Debian'
|
||||
|
||||
- name: Set fact podman_binary (Red Hat family)
|
||||
ansible.builtin.set_fact:
|
||||
podman_binary: /usr/sbin/podman
|
||||
when: ansible_os_family == 'RedHat'
|
||||
|
||||
- name: Create required folders
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "{{ llama_server_home }}/{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ llama_server_user }}"
|
||||
group: "{{ llama_server_group }}"
|
||||
mode: "0750"
|
||||
loop:
|
||||
- models
|
||||
- .cache
|
||||
- .cache/llama.cpp
|
||||
|
||||
- name: "Add SSH Authorized key"
|
||||
become: true
|
||||
ansible.posix.authorized_key:
|
||||
user: "{{ llama_server_user }}"
|
||||
key: "{{ llama_server_pubkey }}"
|
||||
state: "{{ 'present' if llama_server_pubkey is defined and llama_server_pubkey != '' else 'absent' }}"
|
||||
when: llama_server_pubkey is defined and llama_server_pubkey != ""
|
||||
|
||||
- name: SELinux tasks (RedHat only)
|
||||
when: ansible_os_family == 'RedHat'
|
||||
become: true
|
||||
block:
|
||||
- name: Persistently set SELinux context
|
||||
community.general.sefcontext:
|
||||
target: "{{ llama_server_home }}(/.*)?"
|
||||
setype: user_home_dir_t
|
||||
state: present
|
||||
|
||||
- name: Persistently set SELinux context (ssh authorized keys)
|
||||
community.general.sefcontext:
|
||||
target: "{{ llama_server_home }}/.ssh/authorized_keys"
|
||||
setype: ssh_home_t
|
||||
state: present
|
||||
when: llama_server_pubkey is defined and llama_server_pubkey != ""
|
||||
|
||||
- name: Apply SELinux context
|
||||
ansible.builtin.command: restorecon -Rv {{ llama_server_home }}
|
||||
changed_when: false
|
||||
|
||||
- name: Enable sudo access to llama user (temporary)
|
||||
become: true
|
||||
ansible.builtin.lineinfile:
|
||||
path: "/etc/sudoers.d/ansible_{{ llama_server_user }}_tmp"
|
||||
create: true
|
||||
mode: "0440"
|
||||
line: "{{ ansible_facts['user_id'] }} ALL=({{ llama_server_user }}) NOPASSWD: ALL"
|
||||
validate: "visudo -cf %s"
|
||||
|
||||
- name: Enable lingering for llama user
|
||||
become: true
|
||||
ansible.builtin.command: loginctl enable-linger {{ llama_server_user }}
|
||||
register: linger_status
|
||||
changed_when: >-
|
||||
'created' in linger_status.stdout or
|
||||
(linger_status.rc == 0 and not
|
||||
('linger file already exists' in linger_status.stderr or
|
||||
'linger file does not exist' in linger_status.stderr))
|
||||
failed_when:
|
||||
- linger_status.rc != 0 and not
|
||||
('linger file already exists' in linger_status.stderr)
|
||||
|
||||
- name: Get current user's UID
|
||||
ansible.builtin.command: id -u {{ llama_server_user }}
|
||||
changed_when: false
|
||||
register: uid
|
||||
|
||||
- name: Get current user's GID
|
||||
ansible.builtin.command: id -g {{ llama_server_user }}
|
||||
changed_when: false
|
||||
register: gid
|
||||
|
||||
- name: Run tasks as podman user
|
||||
become: true
|
||||
become_user: "{{ llama_server_user }}"
|
||||
block:
|
||||
- name: Pull the requested images
|
||||
containers.podman.podman_image:
|
||||
name: "{{ llama_server_image }}"
|
||||
tag: "{{ llama_server_tag }}"
|
||||
state: present
|
||||
|
||||
- name: Create pods
|
||||
containers.podman.podman_container:
|
||||
name: "llama.cpp"
|
||||
image: "{{ llama_server_image }}:{{ llama_server_tag }}"
|
||||
state: started
|
||||
device: "{{ llama_server_devices | default(omit) }}"
|
||||
env: "{{ llama_server_env | default(omit) }}"
|
||||
ports: "0.0.0.0:{{ llama_server_port }}:8080/tcp"
|
||||
volumes:
|
||||
- "{{ llama_server_home }}/models:/models:Z"
|
||||
- "{{ llama_server_home }}/.cache:/app/.cache:Z"
|
||||
command:
|
||||
- "-t"
|
||||
- "{{ llama_server_argv_threads | default(10) }}"
|
||||
- "-np"
|
||||
- "{{ llama_server_argv_parallel | default(1) }}"
|
||||
- "-b"
|
||||
- "{{ llama_server_argv_batch_size | default(1024) }}"
|
||||
- "-ub"
|
||||
- "{{ llama_server_argv_ubatch_size | default(512) }}"
|
||||
- "-fa"
|
||||
- "{{ llama_server_argv_flash_attn | default('on') }}"
|
||||
- "-ctk"
|
||||
- "{{ llama_server_argv_cache_type_k | default('q4_0') }}"
|
||||
- "-ctv"
|
||||
- "{{ llama_server_argv_cache_type_v | default('q8_0') }}"
|
||||
- "-cram"
|
||||
- "{{ llama_server_argv_cache_reuse | default(-1) }}"
|
||||
- "-sm"
|
||||
- "{{ llama_server_argv_split_mode | default('layer') }}"
|
||||
- "-dev"
|
||||
- "{{ llama_server_argv_devices | default('Vulkan1,Vulkan2') }}"
|
||||
- "-ts"
|
||||
- "{{ llama_server_argv_tensor_split | default('8,12') }}"
|
||||
- "-fit"
|
||||
- "{{ llama_server_argv_fit | default('off') }}"
|
||||
- "-mg"
|
||||
- "{{ llama_server_argv_main_gpu | default(1) }}"
|
||||
- "--mmap"
|
||||
- "--metrics"
|
||||
- "--models-preset"
|
||||
- "/models/.router/models.ini"
|
||||
- "--models-max"
|
||||
- "{{ llama_server_models_max }}"
|
||||
|
||||
cmd_args:
|
||||
- "--userns=keep-id"
|
||||
- "--security-opt=label=disable"
|
||||
|
||||
- name: Create systemd service file for pods
|
||||
become: true
|
||||
ansible.builtin.template:
|
||||
src: podman.service.j2
|
||||
dest: "/etc/systemd/system/podman-llama.cpp.service"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
|
||||
- name: Reload systemd daemon
|
||||
become: true
|
||||
ansible.builtin.systemd:
|
||||
daemon_reload: true
|
||||
|
||||
- name: Enable containers at boot
|
||||
become: true
|
||||
ansible.builtin.systemd:
|
||||
name: "podman-llama.cpp"
|
||||
enabled: true
|
||||
state: "started"
|
||||
|
||||
- name: Install Python packages required for llama server model sync
|
||||
become: true
|
||||
ansible.builtin.package:
|
||||
name: "{{ __llama_sync_system_packages__ }}"
|
||||
state: present
|
||||
vars:
|
||||
__llama_sync_system_packages__: >-
|
||||
{{
|
||||
{
|
||||
'Debian': ['python3-pip', 'python3-packaging', 'python3-venv'],
|
||||
'RedHat': ['python3-pip', 'python3-packaging'],
|
||||
'Archlinux': ['python-pip', 'python-packaging']
|
||||
}[ansible_os_family]
|
||||
}}
|
||||
|
||||
- name: Install Hugging Face hub client in llama server sync virtualenv
|
||||
become: true
|
||||
become_user: "{{ llama_server_user }}"
|
||||
ansible.builtin.pip:
|
||||
name:
|
||||
- pip
|
||||
- setuptools
|
||||
- packaging
|
||||
- huggingface_hub>=0.32.0
|
||||
state: present
|
||||
virtualenv: "{{ llama_server_home }}/.router/.venv"
|
||||
virtualenv_command: "python3 -m venv"
|
||||
|
||||
- name: Create llama server router model directories
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "{{ item.path }}"
|
||||
state: directory
|
||||
owner: "{{ llama_server_user }}"
|
||||
group: "{{ llama_server_group }}"
|
||||
mode: "{{ item.mode }}"
|
||||
loop:
|
||||
- { path: "{{ llama_server_home }}/models", mode: "0750" }
|
||||
- { path: "{{ llama_server_home }}/models/managed", mode: "0750" }
|
||||
- { path: "{{ llama_server_home }}/models/managed-links", mode: "0750" }
|
||||
- { path: "{{ llama_server_home }}/models/.router", mode: "0750" }
|
||||
|
||||
- name: Copy llama server Hugging Face sync helper
|
||||
become: true
|
||||
ansible.builtin.copy:
|
||||
src: scripts/llama_hf_sync.py
|
||||
dest: "{{ llama_server_home }}/models/.router/llama_hf_sync.py"
|
||||
owner: "{{ llama_server_user }}"
|
||||
group: "{{ llama_server_group }}"
|
||||
mode: "0750"
|
||||
|
||||
- name: Write desired llama server model manifest input
|
||||
become: true
|
||||
ansible.builtin.copy:
|
||||
content: "{{ llama_server_models | to_nice_json }}"
|
||||
dest: "{{ llama_server_home }}/models/.router/desired-models.json"
|
||||
owner: "{{ llama_server_user }}"
|
||||
group: "{{ llama_server_group }}"
|
||||
mode: "0640"
|
||||
|
||||
- name: Write global llama server preset options
|
||||
become: true
|
||||
ansible.builtin.copy:
|
||||
content: "{{ llama_server_preset_global | to_nice_json }}"
|
||||
dest: "{{ llama_server_home }}/models/.router/preset-global.json"
|
||||
owner: "{{ llama_server_user }}"
|
||||
group: "{{ llama_server_group }}"
|
||||
mode: "0640"
|
||||
|
||||
- name: Set arguments for llama server model sync
|
||||
ansible.builtin.set_fact:
|
||||
__llama_preset_file__:
|
||||
- "{{ llama_server_home }}/.router/.venv/bin/python"
|
||||
- "{{ llama_server_home }}/models/.router/llama_hf_sync.py"
|
||||
- "--models-file"
|
||||
- "{{ llama_server_home }}/models/.router/desired-models.json"
|
||||
- "--managed-dir"
|
||||
- "{{ llama_server_home }}/models/managed"
|
||||
- "--links-dir"
|
||||
- "{{ llama_server_home }}/models/managed-links"
|
||||
- "--manifest-file"
|
||||
- "{{ llama_server_home }}/models/.router/manifest.json"
|
||||
- "--preset-file"
|
||||
- "{{ llama_server_home }}/models/.router/models.ini"
|
||||
- "--preset-global-file"
|
||||
- "{{ llama_server_home }}/models/.router/preset-global.json"
|
||||
- "--container-links-dir"
|
||||
- "/models/managed-links"
|
||||
|
||||
- name: Run llama server model sync from Hugging Face
|
||||
vars:
|
||||
__llama_sync_argv__: >-
|
||||
{{ __llama_preset_file__
|
||||
+ (['--prune'] if llama_server_models_prune_enabled | default(true) else [])
|
||||
+ (['--dry-run'] if llama_server_models_dry_run | default(false) else [])
|
||||
}}
|
||||
become: true
|
||||
become_user: "{{ llama_server_user }}"
|
||||
ansible.builtin.command:
|
||||
argv: "{{ __llama_sync_argv__ }}"
|
||||
environment:
|
||||
HF_TOKEN: "{{ lookup('env', 'HF_TOKEN') | default('', true) }}"
|
||||
register: __llama_sync_output__
|
||||
changed_when: false
|
||||
|
||||
- name: Parse llama server sync output
|
||||
ansible.builtin.set_fact:
|
||||
__llama_sync_result__: "{{ __llama_sync_output__.stdout | from_json }}"
|
||||
|
||||
- name: Stat llama router preset file (when not in dry-run mode)
|
||||
become: true
|
||||
become_user: "{{ llama_server_user }}"
|
||||
ansible.builtin.stat:
|
||||
path: "{{ llama_server_home }}/models/.router/models.ini"
|
||||
register: __llama_preset_stat__
|
||||
when: not (llama_server_models_dry_run | default(false))
|
||||
|
||||
- name: Ensure router preset file exists when not in dry-run mode
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- __llama_preset_stat__.stat.exists
|
||||
fail_msg: "llama router preset file was not created"
|
||||
when: not (llama_server_models_dry_run | default(false))
|
||||
|
||||
- name: Query llama router model list
|
||||
ansible.builtin.uri:
|
||||
url: "http://127.0.0.1:{{ llama_server_port | default(8080) }}/models?reload=1"
|
||||
method: GET
|
||||
status_code: 200
|
||||
register: __llama_router_models__
|
||||
changed_when: false
|
||||
when:
|
||||
- llama_server_models_sync_enabled | default(true)
|
||||
- not (llama_server_models_dry_run | default(false))
|
||||
|
||||
- name: Validate expected model aliases are available in router
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- item.name in (__llama_router_models__.json.data | map(attribute='id') | list)
|
||||
fail_msg: "Model name '{{ item.name }}' not present in llama router /models output"
|
||||
loop: "{{ __llama_sync_result__.models | default([]) }}"
|
||||
loop_control:
|
||||
label: "{{ item.name }}"
|
||||
when:
|
||||
- llama_server_models_sync_enabled | default(true)
|
||||
- not (llama_server_models_dry_run | default(false))
|
||||
|
||||
- name: Include Llama Exporter setup
|
||||
when: llama_server_exporter_enabled | default(true) | bool
|
||||
ansible.builtin.include_tasks: llama_exporter.yml
|
||||
tags:
|
||||
- roles::llama_server::llama_exporter
|
||||
|
||||
- name: Remove temporary sudo access
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "/etc/sudoers.d/ansible_{{ llama_server_user }}_tmp"
|
||||
state: absent
|
||||
@@ -0,0 +1,35 @@
|
||||
[Unit]
|
||||
Description=LLM Inference Metrics Exporter (llama.cpp + Ollama)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ llama_server_exporter_user }}
|
||||
Group={{ llama_server_exporter_user }}
|
||||
WorkingDirectory={{ llama_server_exporter_install_dir }}
|
||||
Environment="LLAMA_EXPORTER_PORT={{ llama_server_exporter_port }}"
|
||||
Environment="LLAMA_EXPORTER_BIND=0.0.0.0"
|
||||
Environment="LLAMA_EXPORTER_INTERVAL={{ llama_server_exporter_scrape_interval }}"
|
||||
Environment="LLAMA_EXPORTER_TIMEOUT={{ llama_server_exporter_scrape_timeout }}"
|
||||
{% if llama_server_exporter_targets and llama_server_exporter_targets != "" %}
|
||||
Environment="LLAMA_TARGETS={{ llama_server_exporter_targets | to_json }}"
|
||||
{% endif %}
|
||||
Environment="PATH={{ llama_server_exporter_venv_bin }}:/usr/bin:/bin"
|
||||
ExecStart={{ llama_server_exporter_venv_bin }}/python3 {{ llama_server_exporter_script_path }}
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
TimeoutStopSec=10
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=llama_exporter
|
||||
|
||||
# Security hardening
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths={{ llama_server_exporter_install_dir }} /var/log
|
||||
NoNewPrivileges=true
|
||||
PrivateTmp=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,17 @@
|
||||
[Unit]
|
||||
Description=Podman file for llama.cpp server
|
||||
After=local-fs.target network-online.target nvidia-cdi-generator.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User={{ llama_server_user }}
|
||||
Group={{ llama_server_group }}
|
||||
RemainAfterExit=true
|
||||
ExecStart={{ podman_binary }} start podman-llama.cpp
|
||||
ExecStop={{ podman_binary }} stop podman-llama.cpp
|
||||
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
# Internal role variables (non-overridable)
|
||||
|
||||
llama_server_exporter_script_path: "{{ llama_server_exporter_install_dir }}/llama_exporter.py"
|
||||
llama_server_exporter_venv_bin: "{{ llama_server_exporter_venv_path }}/bin"
|
||||
llama_server_exporter_service_name: "llama_exporter"
|
||||
|
||||
# Default llama.cpp targets if none provided
|
||||
llama_server_exporter_targets_default:
|
||||
- name: "llama-cpp-11434"
|
||||
url: "http://localhost:11434"
|
||||
Reference in New Issue
Block a user