feat(vaultwarden): add SMTP configuration options and enhance signup settings
- Introduced SMTP settings for Vaultwarden including host, port, security, and authentication details. - Added variables for signup verification, 2FA settings, password hints, and logging options. - Updated Vaultwarden deployment to utilize new SMTP configurations. - Enhanced Grafana module to support dynamic dashboard and datasource provisioning. - Added LLM proxy configuration for Open Web UI with necessary environment variables.
This commit is contained in:
@@ -0,0 +1,389 @@
|
||||
#!/usr/bin/env python3
|
||||
"""LLM inference metrics exporter for Prometheus.
|
||||
|
||||
Background thread scrapes only loaded models to avoid triggering model loads
|
||||
in llama-server, then the HTTP handler serves the cached data immediately.
|
||||
|
||||
Exposes metrics for every known model from /models; unloaded models show
|
||||
zero values without ever requesting /metrics?model=<unloaded>.
|
||||
|
||||
Uses persistent JSON cache on disk to survive restarts and compute counter
|
||||
deltas for Prometheus rate/irate queries.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import threading
|
||||
import logging
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
POLL_INTERVAL = int(os.environ.get("LLAMA_EXPORTER_INTERVAL", "15"))
|
||||
SCRAPE_TIMEOUT = int(os.environ.get("LLAMA_EXPORTER_TIMEOUT", "5"))
|
||||
CACHE_FILE = os.environ.get(
|
||||
"LLAMA_EXPORTER_CACHE",
|
||||
os.path.join(os.path.dirname(os.path.abspath(__file__)), "llama_exporter_cache.json"),
|
||||
)
|
||||
|
||||
LLAMA_CPP_DEFAULTS = [
|
||||
{"name": "llama.cpp", "url": "http://localhost:11434"},
|
||||
]
|
||||
|
||||
# All llama.cpp metrics we expose per model (counters + gauges).
|
||||
ALL_METRICS = [
|
||||
"llamacpp:prompt_tokens_total",
|
||||
"llamacpp:prompt_seconds_total",
|
||||
"llamacpp:tokens_predicted_total",
|
||||
"llamacpp:tokens_predicted_seconds_total",
|
||||
"llamacpp:n_decode_total",
|
||||
"llamacpp:n_tokens_max",
|
||||
"llamacpp:prompt_tokens_seconds",
|
||||
"llamacpp:predicted_tokens_seconds",
|
||||
"llamacpp:requests_processing",
|
||||
"llamacpp:requests_deferred",
|
||||
"llamacpp:n_busy_slots_per_decode",
|
||||
]
|
||||
|
||||
# Counter metrics that accumulate over time — exposed as _delta for rate() queries.
|
||||
COUNTER_METRICS = {
|
||||
"llamacpp:prompt_tokens_total",
|
||||
"llamacpp:prompt_seconds_total",
|
||||
"llamacpp:tokens_predicted_total",
|
||||
"llamacpp:tokens_predicted_seconds_total",
|
||||
"llamacpp:n_decode_total",
|
||||
}
|
||||
|
||||
|
||||
class Cache:
|
||||
"""Persistent, thread-safe metrics cache with delta computation."""
|
||||
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self._data = {} # { model_id: { metric_name: value } }
|
||||
self._known_models = set()
|
||||
self._health = None
|
||||
self._loaded = {}
|
||||
|
||||
# Previous state for delta computation: { model_id: { metric_name: value } }
|
||||
self._prev = {}
|
||||
self._prev_timestamp = 0.0
|
||||
self._deltas = {}
|
||||
|
||||
# Load persisted state
|
||||
self._load_cache()
|
||||
|
||||
def _load_cache(self):
|
||||
"""Load previous scrape state from disk."""
|
||||
try:
|
||||
with open(CACHE_FILE, "r") as f:
|
||||
state = json.load(f)
|
||||
self._prev = state.get("models", {})
|
||||
self._prev_timestamp = state.get("timestamp", 0.0)
|
||||
logger.info("Loaded cache from %s (timestamp=%s, models=%d)",
|
||||
CACHE_FILE, self._prev_timestamp, len(self._prev))
|
||||
except FileNotFoundError:
|
||||
logger.info("No cache file found at %s", CACHE_FILE)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to load cache: %s", e)
|
||||
|
||||
def _save_cache(self, current_data, known_models, loaded):
|
||||
"""Save current scrape state to disk."""
|
||||
try:
|
||||
state = {
|
||||
"timestamp": time.time(),
|
||||
"known": list(known_models),
|
||||
"loaded": list(loaded),
|
||||
"models": {},
|
||||
}
|
||||
for mid, metrics in current_data.items():
|
||||
state["models"][mid] = dict(metrics)
|
||||
tmp = CACHE_FILE + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(state, f)
|
||||
os.replace(tmp, CACHE_FILE)
|
||||
except Exception as e:
|
||||
logger.error("Failed to save cache: %s", e)
|
||||
|
||||
def _compute_deltas_for_data(self, current_data):
|
||||
"""Compute deltas given current data and stored previous state.
|
||||
|
||||
Only counter metrics get deltas. On counter reset (value went backward),
|
||||
delta is silently 0 (no entry added).
|
||||
"""
|
||||
deltas = {}
|
||||
if not self._prev:
|
||||
return deltas
|
||||
|
||||
for mid, prev_metrics in self._prev.items():
|
||||
mid_deltas = {}
|
||||
cur_metrics = current_data.get(mid, {})
|
||||
for mname, prev_val in prev_metrics.items():
|
||||
# Only compute deltas for counter metrics
|
||||
if mname not in COUNTER_METRICS:
|
||||
continue
|
||||
cur_val = cur_metrics.get(mname)
|
||||
if cur_val is None:
|
||||
continue
|
||||
diff = cur_val - prev_val
|
||||
# Counter reset: value went backward, delta is 0 (wrapped)
|
||||
if diff < 0:
|
||||
continue
|
||||
if diff > 0:
|
||||
mid_deltas[mname] = diff
|
||||
if mid_deltas:
|
||||
deltas[mid] = mid_deltas
|
||||
return deltas
|
||||
|
||||
def update(self, models_data, metric_lines, known_models=None):
|
||||
"""Called by the background thread after a successful scrape cycle."""
|
||||
with self._lock:
|
||||
# Discover model IDs and loaded status from /models endpoint
|
||||
known = set()
|
||||
loaded = {}
|
||||
if models_data and isinstance(models_data, dict):
|
||||
model_list = models_data.get("data", [])
|
||||
for m in model_list:
|
||||
if not isinstance(m, dict):
|
||||
continue
|
||||
mid = m.get("id", "unknown")
|
||||
known.add(mid)
|
||||
status_data = m.get("status", {})
|
||||
if isinstance(status_data, dict) and status_data.get("value") == "loaded":
|
||||
loaded[mid] = 1.0
|
||||
self._health = {"status": "ok", "model": mid}
|
||||
|
||||
if not known:
|
||||
self._health = {"status": "error", "model": "unknown"}
|
||||
|
||||
# Build new metrics dict for all known models
|
||||
new_data = {}
|
||||
for mid in known:
|
||||
new_data[mid] = {}
|
||||
|
||||
# Apply parsed metric values from /metrics?model=<id>
|
||||
for metric_name, model_id, value in metric_lines:
|
||||
if model_id in known:
|
||||
new_data[model_id][metric_name] = value
|
||||
|
||||
# Ensure every known model has all ALL_METRICS entries
|
||||
for mid in known:
|
||||
for mname in ALL_METRICS:
|
||||
if mname not in new_data[mid]:
|
||||
new_data[mid][mname] = 0.0
|
||||
|
||||
# Previously known models no longer in the list get zeroed out
|
||||
for mid in self._known_models - known:
|
||||
new_data[mid] = {m: 0.0 for m in ALL_METRICS}
|
||||
|
||||
# Compute deltas before updating previous state
|
||||
deltas = self._compute_deltas_for_data(new_data)
|
||||
|
||||
# Save previous state for next cycle
|
||||
self._save_cache(new_data, known, loaded)
|
||||
|
||||
# Update state
|
||||
self._prev = {mid: dict(metrics) for mid, metrics in new_data.items()}
|
||||
self._known_models = known
|
||||
self._data = new_data
|
||||
self._loaded = loaded
|
||||
self._deltas = deltas
|
||||
|
||||
def snapshot(self):
|
||||
"""Return a frozen copy of the current cache state including deltas."""
|
||||
with self._lock:
|
||||
return {
|
||||
"data": {k: dict(v) for k, v in self._data.items()},
|
||||
"known": set(self._known_models),
|
||||
"health": dict(self._health) if self._health else {"status": "error", "model": "unknown"},
|
||||
"loaded": dict(self._loaded),
|
||||
"deltas": {k: dict(v) for k, v in self._deltas.items()},
|
||||
}
|
||||
|
||||
|
||||
cache = Cache()
|
||||
|
||||
|
||||
def _fetch_json(url, timeout=SCRAPE_TIMEOUT):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"Accept": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.debug("Failed to fetch %s: %s", url, e)
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_text(url, timeout=SCRAPE_TIMEOUT):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"Accept": "text/plain"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.read().decode("utf-8")
|
||||
except Exception as e:
|
||||
logger.debug("Failed to fetch %s: %s", url, e)
|
||||
return None
|
||||
|
||||
|
||||
def _scrape_cycle():
|
||||
"""One full scrape cycle: discover models, then scrape metrics per model."""
|
||||
targets = []
|
||||
env_targets = os.environ.get("LLAMA_TARGETS", "")
|
||||
if env_targets:
|
||||
try:
|
||||
targets = json.loads(env_targets)
|
||||
except json.JSONDecodeError:
|
||||
targets = []
|
||||
if not targets:
|
||||
targets = LLAMA_CPP_DEFAULTS
|
||||
|
||||
all_metric_lines = []
|
||||
models_data = None
|
||||
known_models = {} # { model_id: base_url, ... }
|
||||
|
||||
for target in targets:
|
||||
base_url = target["url"].rstrip("/")
|
||||
|
||||
# Fetch /models to discover models and their status
|
||||
models_data = _fetch_json(f"{base_url}/models")
|
||||
|
||||
if models_data and isinstance(models_data, dict):
|
||||
model_list = models_data.get("data", [])
|
||||
for m in model_list:
|
||||
if not isinstance(m, dict) or "id" not in m:
|
||||
continue
|
||||
model_id = m["id"]
|
||||
known_models[model_id] = base_url
|
||||
# Only scrape metrics from loaded models to avoid triggering loads
|
||||
status_data = m.get("status", {})
|
||||
if isinstance(status_data, dict) and status_data.get("value") != "loaded":
|
||||
continue
|
||||
# Scrape /metrics for loaded models only
|
||||
metrics_url = f"{base_url}/metrics?model={model_id}"
|
||||
body = _fetch_text(metrics_url)
|
||||
if body:
|
||||
for line in body.splitlines():
|
||||
parsed = _parse_metric_line(line)
|
||||
if parsed:
|
||||
metric_name, metric_value = parsed
|
||||
if metric_name in ALL_METRICS:
|
||||
all_metric_lines.append((metric_name, model_id, metric_value))
|
||||
else:
|
||||
logger.debug("No metrics body for model %s", model_id)
|
||||
|
||||
# Update the shared cache
|
||||
cache.update(models_data, all_metric_lines, known_models)
|
||||
|
||||
|
||||
def _parse_metric_line(line):
|
||||
"""Parse a single Prometheus metric line. Returns (name, value) or None."""
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
return None
|
||||
try:
|
||||
# Handle lines with labels: metric_name{label="val"} value
|
||||
if "{" in line:
|
||||
parts = line.split("{")
|
||||
name = parts[0].strip()
|
||||
rest = parts[1]
|
||||
# value is the last token after the closing }
|
||||
value = rest.rsplit("}", 1)[-1].strip().split()[-1] if "}" in rest else rest.strip()
|
||||
else:
|
||||
parts = line.split()
|
||||
name = parts[0]
|
||||
value = parts[1] if len(parts) >= 2 else "1"
|
||||
# Try to parse as float
|
||||
float(value)
|
||||
return (name, float(value))
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def _background_scrape():
|
||||
"""Background thread: periodically scrape and update cache."""
|
||||
logger.info("Background scraper started (interval=%ds)", POLL_INTERVAL)
|
||||
# Do one immediate scrape on startup
|
||||
_scrape_cycle()
|
||||
while True:
|
||||
try:
|
||||
time.sleep(POLL_INTERVAL)
|
||||
_scrape_cycle()
|
||||
except Exception as e:
|
||||
logger.error("Scrape cycle error: %s", e)
|
||||
|
||||
|
||||
class MetricsHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
if self.path != "/metrics":
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
snap = cache.snapshot()
|
||||
lines = []
|
||||
|
||||
def _fmt(metric_name, model_id, value):
|
||||
return metric_name + '{' + 'server="llama-cpp-11434",model="' + model_id + '"} ' + str(value)
|
||||
|
||||
# Health metric
|
||||
health = snap["health"]
|
||||
status = health.get("status", "error")
|
||||
health_model = health.get("model", "unknown")
|
||||
health_val = 1.0 if status == "ok" else 0.0
|
||||
lines.append(_fmt("llama_server_health", health_model, health_val))
|
||||
|
||||
# Loaded metrics
|
||||
for mid in snap["loaded"]:
|
||||
lines.append(_fmt("llama_models_loaded", mid, snap["loaded"][mid]))
|
||||
|
||||
# Per-model metrics from cache (absolute values)
|
||||
for mid in sorted(snap["data"]):
|
||||
metrics = snap["data"][mid]
|
||||
for mname in ALL_METRICS:
|
||||
value = metrics.get(mname, 0.0)
|
||||
lines.append(_fmt(mname, mid, value))
|
||||
|
||||
# Per-model delta metrics (counters as change since last scrape)
|
||||
deltas = snap.get("deltas", {})
|
||||
for mid in sorted(deltas):
|
||||
delta_metrics = deltas[mid]
|
||||
for mname in sorted(delta_metrics):
|
||||
value = delta_metrics[mname]
|
||||
lines.append(_fmt(mname + "_delta", mid, value))
|
||||
|
||||
body = "\n".join(lines) + "\n" if lines else "# no metrics available\n"
|
||||
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
self.end_headers()
|
||||
self.wfile.write(body.encode("utf-8"))
|
||||
|
||||
def log_message(self, format, *args):
|
||||
logger.debug("%s - - %s", self.address_string(), format % args)
|
||||
|
||||
|
||||
def main():
|
||||
port = int(os.environ.get("LLAMA_EXPORTER_PORT", "9550"))
|
||||
host = os.environ.get("LLAMA_EXPORTER_BIND", "0.0.0.0")
|
||||
|
||||
# Start background scraper thread
|
||||
scraper = threading.Thread(target=_background_scrape, daemon=True)
|
||||
scraper.start()
|
||||
|
||||
# Start HTTP server
|
||||
server = HTTPServer((host, port), MetricsHandler)
|
||||
logger.info("Starting Llama Exporter on %s:%d (interval=%ds, timeout=%ds, cache=%s)",
|
||||
host, port, POLL_INTERVAL, SCRAPE_TIMEOUT, CACHE_FILE)
|
||||
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Shutting down")
|
||||
server.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Synchronize llama.cpp models from Hugging Face into a managed local directory."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import configparser
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from huggingface_hub import HfApi, snapshot_download
|
||||
|
||||
|
||||
def _sanitize_slug(value: str) -> str:
|
||||
slug = re.sub(r"[^a-zA-Z0-9._-]+", "-", value.strip())
|
||||
slug = slug.strip("-._")
|
||||
return slug.lower() or "model"
|
||||
|
||||
|
||||
def _resolve_repo_file(api: HfApi, model: dict[str, Any]) -> str:
|
||||
repo_files = api.list_repo_files(repo_id=model["model_id"], revision=model["revision"], repo_type="model")
|
||||
|
||||
hf_file = model.get("hf_file")
|
||||
if hf_file:
|
||||
if hf_file in repo_files:
|
||||
return hf_file
|
||||
raise RuntimeError(
|
||||
f"Requested hf_file '{hf_file}' was not found in repo "
|
||||
f"{model['model_id']}@{model['revision']}"
|
||||
)
|
||||
|
||||
quant_upper = (model.get("quant") or "").upper()
|
||||
ggufs = [item for item in repo_files if item.lower().endswith(".gguf")]
|
||||
matches = [item for item in ggufs if quant_upper in Path(item).name.upper()]
|
||||
if matches:
|
||||
return sorted(matches, key=lambda name: (len(Path(name).name), name))[0]
|
||||
|
||||
raise RuntimeError(
|
||||
f"No GGUF file containing quant '{model.get('quant')}' found in "
|
||||
f"repo {model['model_id']}@{model['revision']}. Set hf_file explicitly if needed."
|
||||
)
|
||||
|
||||
|
||||
def _load_models(models_file: Path) -> list[dict[str, Any]]:
|
||||
data = json.loads(models_file.read_text(encoding="utf-8"))
|
||||
if not isinstance(data, list):
|
||||
raise RuntimeError("models file must contain a JSON array")
|
||||
|
||||
normalized: list[dict[str, Any]] = []
|
||||
for item in data:
|
||||
if not isinstance(item, dict):
|
||||
raise RuntimeError("each model entry must be an object")
|
||||
if item.get("enable", True) is False:
|
||||
continue
|
||||
|
||||
model_id = str(item.get("model_id", "")).strip()
|
||||
quant = str(item.get("quant", "")).strip()
|
||||
revision = str(item.get("revision", "")).strip()
|
||||
hf_file = str(item.get("hf_file", "")).strip() or None
|
||||
name = str(item.get("name", "")).strip()
|
||||
preset = item.get("preset", {})
|
||||
|
||||
if preset is None:
|
||||
preset = {}
|
||||
if not isinstance(preset, dict):
|
||||
raise RuntimeError(f"preset must be an object for model '{name or model_id or 'unknown'}'")
|
||||
|
||||
if not name:
|
||||
raise RuntimeError(f"name is required for model '{model_id or 'unknown'}'")
|
||||
if not model_id:
|
||||
raise RuntimeError("model_id is required for all models")
|
||||
if not quant and not hf_file:
|
||||
raise RuntimeError(f"quant or hf_file is required for model '{model_id}'")
|
||||
if not revision:
|
||||
raise RuntimeError(f"revision is required for model '{model_id}'")
|
||||
|
||||
normalized.append(
|
||||
{
|
||||
"model_id": model_id,
|
||||
"quant": quant,
|
||||
"revision": revision,
|
||||
"hf_file": hf_file,
|
||||
"name": name,
|
||||
"preset": preset,
|
||||
}
|
||||
)
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def _load_preset_global(preset_global_file: Path | None) -> dict[str, Any]:
|
||||
if preset_global_file is None:
|
||||
return {}
|
||||
|
||||
if not preset_global_file.exists():
|
||||
return {}
|
||||
|
||||
data = json.loads(preset_global_file.read_text(encoding="utf-8"))
|
||||
if data is None:
|
||||
return {}
|
||||
if not isinstance(data, dict):
|
||||
raise RuntimeError("global preset options must be a JSON object")
|
||||
return data
|
||||
|
||||
|
||||
def _to_preset_value(value: Any) -> str:
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, (int, float, str)):
|
||||
return str(value)
|
||||
return json.dumps(value, separators=(",", ":"))
|
||||
|
||||
|
||||
def _normalize_preset_options(options: dict[str, Any], scope: str) -> dict[str, str]:
|
||||
normalized: dict[str, str] = {}
|
||||
for key, value in options.items():
|
||||
key_str = str(key).strip()
|
||||
if not key_str:
|
||||
raise RuntimeError(f"{scope} preset option keys must be non-empty")
|
||||
if value is None:
|
||||
continue
|
||||
normalized[key_str] = _to_preset_value(value)
|
||||
return normalized
|
||||
|
||||
|
||||
def _download_model(model: dict[str, Any], target_dir: Path, repo_file: str, dry_run: bool) -> None:
|
||||
if dry_run:
|
||||
return
|
||||
|
||||
snapshot_download(
|
||||
repo_id=model["model_id"],
|
||||
revision=model["revision"],
|
||||
local_dir=str(target_dir),
|
||||
allow_patterns=[repo_file],
|
||||
local_dir_use_symlinks=False,
|
||||
)
|
||||
|
||||
|
||||
def _write_preset(preset_file: Path, entries: list[dict[str, Any]], global_options: dict[str, Any]) -> None:
|
||||
parser = configparser.ConfigParser(interpolation=None)
|
||||
parser.optionxform = str
|
||||
parser["*"] = _normalize_preset_options(global_options, "global")
|
||||
|
||||
for entry in entries:
|
||||
model_options = _normalize_preset_options(entry.get("preset", {}), f"model '{entry['name']}'")
|
||||
model_options["model"] = entry["container_model_path"]
|
||||
parser[entry["name"]] = model_options
|
||||
|
||||
preset_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
with preset_file.open("w", encoding="utf-8") as fh:
|
||||
# Router preset format supports top-level version key.
|
||||
fh.write("version = 1\n\n")
|
||||
parser.write(fh)
|
||||
|
||||
|
||||
def _prune_unmanaged(managed_dir: Path, links_dir: Path, keep_slugs: set[str], dry_run: bool) -> tuple[list[str], list[str]]:
|
||||
pruned_dirs: list[str] = []
|
||||
pruned_links: list[str] = []
|
||||
|
||||
if managed_dir.exists():
|
||||
for child in managed_dir.iterdir():
|
||||
if not child.is_dir():
|
||||
continue
|
||||
if child.name == ".router":
|
||||
continue
|
||||
if child.name not in keep_slugs:
|
||||
pruned_dirs.append(child.name)
|
||||
if not dry_run:
|
||||
shutil.rmtree(child)
|
||||
|
||||
if links_dir.exists():
|
||||
for child in links_dir.iterdir():
|
||||
if child.suffix != ".gguf":
|
||||
continue
|
||||
slug = child.stem
|
||||
if slug not in keep_slugs:
|
||||
pruned_links.append(child.name)
|
||||
if not dry_run:
|
||||
child.unlink(missing_ok=True)
|
||||
|
||||
return pruned_dirs, pruned_links
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Sync Hugging Face GGUF models for llama.cpp router mode")
|
||||
parser.add_argument("--models-file", required=True)
|
||||
parser.add_argument("--managed-dir", required=True)
|
||||
parser.add_argument("--links-dir", required=True)
|
||||
parser.add_argument("--manifest-file", required=True)
|
||||
parser.add_argument("--preset-file", required=True)
|
||||
parser.add_argument("--preset-global-file")
|
||||
parser.add_argument("--container-links-dir", required=True)
|
||||
parser.add_argument("--prune", action="store_true")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
models_file = Path(args.models_file)
|
||||
managed_dir = Path(args.managed_dir)
|
||||
links_dir = Path(args.links_dir)
|
||||
manifest_file = Path(args.manifest_file)
|
||||
preset_file = Path(args.preset_file)
|
||||
preset_global_file = Path(args.preset_global_file) if args.preset_global_file else None
|
||||
|
||||
managed_dir.mkdir(parents=True, exist_ok=True)
|
||||
links_dir.mkdir(parents=True, exist_ok=True)
|
||||
manifest_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
preset_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
models = _load_models(models_file)
|
||||
global_options = _load_preset_global(preset_global_file)
|
||||
api = HfApi()
|
||||
entries: list[dict[str, Any]] = []
|
||||
|
||||
for model in models:
|
||||
slug = _sanitize_slug(model["name"])
|
||||
target_dir = managed_dir / slug
|
||||
target_dir.mkdir(parents=True, exist_ok=True)
|
||||
repo_file = _resolve_repo_file(api, model)
|
||||
|
||||
_download_model(model, target_dir, repo_file, args.dry_run)
|
||||
selected_file = target_dir / repo_file
|
||||
if not args.dry_run and not selected_file.exists():
|
||||
raise RuntimeError(f"Expected downloaded file not found: {selected_file}")
|
||||
|
||||
link_path = links_dir / f"{slug}.gguf"
|
||||
container_model_path = f"{args.container_links_dir.rstrip('/')}/{slug}.gguf"
|
||||
|
||||
if not args.dry_run:
|
||||
if link_path.exists() or link_path.is_symlink():
|
||||
link_path.unlink()
|
||||
# Keep symlink targets relative so they remain valid inside the
|
||||
# container-mounted /models tree.
|
||||
relative_target = os.path.relpath(selected_file, start=link_path.parent)
|
||||
link_path.symlink_to(relative_target)
|
||||
|
||||
entries.append(
|
||||
{
|
||||
"name": model["name"],
|
||||
"slug": slug,
|
||||
"model_id": model["model_id"],
|
||||
"revision": model["revision"],
|
||||
"quant": model["quant"],
|
||||
"hf_file": model.get("hf_file"),
|
||||
"preset": model.get("preset", {}),
|
||||
"repo_file": repo_file,
|
||||
"selected_file": str(selected_file),
|
||||
"link_path": str(link_path),
|
||||
"container_model_path": container_model_path,
|
||||
}
|
||||
)
|
||||
|
||||
pruned_dirs: list[str] = []
|
||||
pruned_links: list[str] = []
|
||||
if args.prune:
|
||||
keep_slugs = {entry["slug"] for entry in entries}
|
||||
pruned_dirs, pruned_links = _prune_unmanaged(managed_dir, links_dir, keep_slugs, args.dry_run)
|
||||
|
||||
result = {
|
||||
"models": entries,
|
||||
"dry_run": args.dry_run,
|
||||
"prune": args.prune,
|
||||
"pruned_dirs": pruned_dirs,
|
||||
"pruned_links": pruned_links,
|
||||
}
|
||||
|
||||
if not args.dry_run:
|
||||
manifest_file.write_text(json.dumps(result, indent=2), encoding="utf-8")
|
||||
_write_preset(preset_file, entries, global_options)
|
||||
|
||||
print(json.dumps(result))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except Exception as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for llama_exporter cache persistence and delta computation."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import shutil
|
||||
import unittest
|
||||
|
||||
# Setup temp cache file before importing the module
|
||||
TEMP_DIR = tempfile.mkdtemp()
|
||||
os.environ["LLAMA_EXPORTER_CACHE"] = os.path.join(TEMP_DIR, "test_cache.json")
|
||||
|
||||
# Import from parent directory (scripts)
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from llama_exporter import Cache
|
||||
|
||||
|
||||
class TestCachePersistenceAndDeltas(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
# Ensure temp dir exists (pytest may clean it between test methods)
|
||||
os.makedirs(TEMP_DIR, exist_ok=True)
|
||||
self.cache = Cache()
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(TEMP_DIR, ignore_errors=True)
|
||||
|
||||
def test_cold_start_no_deltas(self):
|
||||
"""First scrape should produce no deltas since there's no previous state."""
|
||||
models_data = {
|
||||
"data": [
|
||||
{"id": "model-v1", "status": {"value": "loaded"}},
|
||||
{"id": "model-v2", "status": {"value": "unloaded"}},
|
||||
]
|
||||
}
|
||||
metric_lines = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 100.0),
|
||||
("llamacpp:prompt_seconds_total", "model-v1", 2.5),
|
||||
("llamacpp:n_tokens_max", "model-v1", 4096.0),
|
||||
]
|
||||
known_models = {"model-v1": "http://localhost:11434", "model-v2": "http://localhost:11434"}
|
||||
|
||||
self.cache.update(models_data, metric_lines, known_models)
|
||||
|
||||
snap = self.cache.snapshot()
|
||||
# No deltas on first scrape
|
||||
self.assertEqual(snap["deltas"], {})
|
||||
# Absolute values are correct
|
||||
self.assertEqual(snap["data"]["model-v1"]["llamacpp:prompt_tokens_total"], 100.0)
|
||||
# Unloaded model has zeros
|
||||
self.assertEqual(snap["data"]["model-v2"]["llamacpp:prompt_tokens_total"], 0.0)
|
||||
# Only loaded model is in loaded set
|
||||
self.assertIn("model-v1", snap["loaded"])
|
||||
self.assertNotIn("model-v2", snap["loaded"])
|
||||
|
||||
def test_delta_computation(self):
|
||||
"""Second scrape should produce deltas from first scrape."""
|
||||
models_data = {
|
||||
"data": [
|
||||
{"id": "model-v1", "status": {"value": "loaded"}},
|
||||
]
|
||||
}
|
||||
|
||||
# First scrape
|
||||
metric_lines_1 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 100.0),
|
||||
("llamacpp:prompt_seconds_total", "model-v1", 2.5),
|
||||
]
|
||||
known_models_1 = {"model-v1": "http://localhost:11434"}
|
||||
self.cache.update(models_data, metric_lines_1, known_models_1)
|
||||
|
||||
# Second scrape - values increased
|
||||
metric_lines_2 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 250.0),
|
||||
("llamacpp:prompt_seconds_total", "model-v1", 5.0),
|
||||
]
|
||||
self.cache.update(models_data, metric_lines_2, known_models_1)
|
||||
|
||||
snap = self.cache.snapshot()
|
||||
# Deltas should be computed
|
||||
self.assertIn("model-v1", snap["deltas"])
|
||||
self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_tokens_total"], 150.0)
|
||||
self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_seconds_total"], 2.5)
|
||||
|
||||
def test_counter_reset(self):
|
||||
"""If a counter value goes backward, delta should be 0."""
|
||||
models_data = {
|
||||
"data": [
|
||||
{"id": "model-v1", "status": {"value": "loaded"}},
|
||||
]
|
||||
}
|
||||
|
||||
# First scrape
|
||||
metric_lines_1 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 100.0),
|
||||
]
|
||||
known_models_1 = {"model-v1": "http://localhost:11434"}
|
||||
self.cache.update(models_data, metric_lines_1, known_models_1)
|
||||
|
||||
# Second scrape - counter reset (model was reloaded)
|
||||
metric_lines_2 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 10.0),
|
||||
]
|
||||
self.cache.update(models_data, metric_lines_2, known_models_1)
|
||||
|
||||
snap = self.cache.snapshot()
|
||||
# On counter reset, delta is implicitly 0 (no entry in deltas)
|
||||
self.assertNotIn("model-v1", snap["deltas"])
|
||||
|
||||
def test_gauge_metrics_no_delta(self):
|
||||
"""Gauge metrics should not get delta entries (only counters do)."""
|
||||
models_data = {
|
||||
"data": [
|
||||
{"id": "model-v1", "status": {"value": "loaded"}},
|
||||
]
|
||||
}
|
||||
|
||||
# First scrape
|
||||
metric_lines_1 = [
|
||||
("llamacpp:n_tokens_max", "model-v1", 4096.0),
|
||||
("llamacpp:requests_processing", "model-v1", 2.0),
|
||||
]
|
||||
known_models_1 = {"model-v1": "http://localhost:11434"}
|
||||
self.cache.update(models_data, metric_lines_1, known_models_1)
|
||||
|
||||
# Second scrape
|
||||
metric_lines_2 = [
|
||||
("llamacpp:n_tokens_max", "model-v1", 8192.0),
|
||||
("llamacpp:requests_processing", "model-v1", 5.0),
|
||||
]
|
||||
self.cache.update(models_data, metric_lines_2, known_models_1)
|
||||
|
||||
snap = self.cache.snapshot()
|
||||
# Gauge metrics should not have deltas
|
||||
self.assertNotIn("model-v1", snap["deltas"])
|
||||
|
||||
def test_persistence_across_restarts(self):
|
||||
"""Delta computation should work across cache reloads from disk."""
|
||||
models_data = {
|
||||
"data": [
|
||||
{"id": "model-v1", "status": {"value": "loaded"}},
|
||||
]
|
||||
}
|
||||
|
||||
# First scrape (first process instance)
|
||||
metric_lines_1 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 100.0),
|
||||
("llamacpp:prompt_seconds_total", "model-v1", 2.5),
|
||||
]
|
||||
known_models_1 = {"model-v1": "http://localhost:11434"}
|
||||
self.cache.update(models_data, metric_lines_1, known_models_1)
|
||||
|
||||
# Simulate restart: create new Cache instance (loads from disk)
|
||||
self.cache._save_cache(self.cache._data, self.cache._known_models, self.cache._loaded)
|
||||
|
||||
# Create a new cache instance (simulates restart)
|
||||
new_cache = Cache()
|
||||
|
||||
# Second scrape after restart
|
||||
metric_lines_2 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 300.0),
|
||||
("llamacpp:prompt_seconds_total", "model-v1", 8.0),
|
||||
]
|
||||
known_models_2 = {"model-v1": "http://localhost:11434"}
|
||||
new_cache.update(models_data, metric_lines_2, known_models_2)
|
||||
|
||||
snap = new_cache.snapshot()
|
||||
# Deltas should be computed using persisted previous state
|
||||
self.assertIn("model-v1", snap["deltas"])
|
||||
self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_tokens_total"], 200.0)
|
||||
self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_seconds_total"], 5.5)
|
||||
|
||||
def test_unloaded_model_zero_values(self):
|
||||
"""Unloaded models should appear with zero values but never trigger /metrics calls."""
|
||||
models_data = {
|
||||
"data": [
|
||||
{"id": "model-v1", "status": {"value": "loaded"}},
|
||||
{"id": "model-v2", "status": {"value": "unloaded"}},
|
||||
{"id": "model-v3", "status": {"value": "unloaded"}},
|
||||
]
|
||||
}
|
||||
metric_lines = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 50.0),
|
||||
]
|
||||
known_models = {
|
||||
"model-v1": "http://localhost:11434",
|
||||
"model-v2": "http://localhost:11434",
|
||||
"model-v3": "http://localhost:11434",
|
||||
}
|
||||
|
||||
self.cache.update(models_data, metric_lines, known_models)
|
||||
|
||||
snap = self.cache.snapshot()
|
||||
# Loaded model has its actual value
|
||||
self.assertEqual(snap["data"]["model-v1"]["llamacpp:prompt_tokens_total"], 50.0)
|
||||
# Unloaded models have zeros for all metrics
|
||||
for m in ["model-v2", "model-v3"]:
|
||||
self.assertEqual(snap["data"][m]["llamacpp:prompt_tokens_total"], 0.0)
|
||||
self.assertEqual(snap["data"][m]["llamacpp:n_tokens_max"], 0.0)
|
||||
|
||||
def test_removed_model_zeroed(self):
|
||||
"""Models removed from llama-server should be zeroed out."""
|
||||
models_data = {
|
||||
"data": [
|
||||
{"id": "model-v1", "status": {"value": "loaded"}},
|
||||
]
|
||||
}
|
||||
|
||||
# First scrape with two models
|
||||
metric_lines_1 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 100.0),
|
||||
]
|
||||
known_models_1 = {"model-v1": "http://localhost:11434"}
|
||||
self.cache.update(models_data, metric_lines_1, known_models_1)
|
||||
|
||||
# Second scrape with only model-v1 (model-v2 was removed)
|
||||
metric_lines_2 = [
|
||||
("llamacpp:prompt_tokens_total", "model-v1", 200.0),
|
||||
]
|
||||
known_models_2 = {"model-v1": "http://localhost:11434"}
|
||||
self.cache.update(models_data, metric_lines_2, known_models_2)
|
||||
|
||||
snap = self.cache.snapshot()
|
||||
# model-v1 delta should be 100
|
||||
self.assertEqual(snap["deltas"]["model-v1"]["llamacpp:prompt_tokens_total"], 100.0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user