Add GPU monitoring services and configurations
- Create systemd service templates for llama_exporter, nvidia_exporter, and podman. - Add Prometheus configuration template for GPU metrics scraping. - Introduce variables for GPU monitoring role in main.yml. - Implement tasks for syncing llama models, including user setup and package installation. - Update podman tasks to ensure proper sudo access and lingering for the podman user. - Modify inventory to include gpu_monitoring group. - Add Terraform modules for deploying Vaultwarden, including ingress and service configurations. - Create Grafana dashboard for real-time GPU monitoring metrics. - Update secrets and environment files to include Vaultwarden admin token.
This commit is contained in:
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""LLM inference metrics exporter for Prometheus.
|
||||
|
||||
Scrape targets:
|
||||
- llama.cpp HTTP API servers (standard /metrics or custom health)
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import logging
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from http.server import HTTPServer, BaseHTTPRequestHandler
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
POLL_INTERVAL = int(os.environ.get("LLAMA_EXPORTER_INTERVAL", "15"))
|
||||
SCRAPE_TIMEOUT = int(os.environ.get("LLAMA_EXPORTER_TIMEOUT", "5"))
|
||||
|
||||
LLAMA_CPP_DEFAULTS = [
|
||||
{"name": "llama-cpp-8012", "url": "http://localhost:8012"},
|
||||
]
|
||||
|
||||
|
||||
def _fetch_json(url, timeout=SCRAPE_TIMEOUT):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"Accept": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
except Exception as e:
|
||||
logger.debug("Failed to fetch %s: %s", url, e)
|
||||
return None
|
||||
|
||||
|
||||
def _collect_llama_cpp(target):
|
||||
metrics = {}
|
||||
|
||||
def _set_metric(metric_name, labels, value):
|
||||
label_key = tuple(sorted(labels.items()))
|
||||
metrics[(metric_name, label_key)] = value
|
||||
|
||||
base_url = target["url"].rstrip("/")
|
||||
|
||||
# Check health endpoint
|
||||
health_url = f"{base_url}/health"
|
||||
health = _fetch_json(health_url)
|
||||
if health and isinstance(health, dict):
|
||||
status = health.get("status", "unknown")
|
||||
_set_metric("llama_server_health", {"server": target["name"], "model": health.get("model", "unknown")}, 1.0 if status == "ok" else 0.0)
|
||||
else:
|
||||
_set_metric("llama_server_health", {"server": target["name"], "model": "unknown"}, 0.0)
|
||||
|
||||
# Check /metrics endpoint for llama.cpp built-in metrics
|
||||
metrics_url = f"{base_url}/metrics"
|
||||
try:
|
||||
req = urllib.request.Request(metrics_url, headers={"Accept": "text/plain"})
|
||||
with urllib.request.urlopen(req, timeout=SCRAPE_TIMEOUT) as resp:
|
||||
body = resp.read().decode("utf-8")
|
||||
for line in body.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
# Parse simple key value pairs
|
||||
if "{" in line:
|
||||
key_part = line.split("{")[0]
|
||||
labels_part = line.split("{")[1].rstrip("}").split("}")[0] if "}" in line else ""
|
||||
value = line.split()[-1] if line.split() else "1"
|
||||
label_str = ""
|
||||
if labels_part:
|
||||
label_str = ",".join(f'{{{k}="{v}"}}' for k, v in (
|
||||
pair.split("=") for pair in labels_part.split(",") if "=" in pair
|
||||
))
|
||||
_set_metric(key_part.strip(), {"server": target["name"], "raw": labels_part}, float(value) if value.replace(".", "").replace("-", "").isdigit() else value)
|
||||
else:
|
||||
parts = line.split()
|
||||
if len(parts) >= 2:
|
||||
try:
|
||||
float(parts[1])
|
||||
_set_metric(parts[0], {"server": target["name"]}, parts[1])
|
||||
except ValueError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.debug("Failed to scrape %s/metrics: %s", metrics_url, e)
|
||||
|
||||
# Check /model endpoint for model info
|
||||
model_url = f"{base_url}/models"
|
||||
models = _fetch_json(model_url)
|
||||
if models and isinstance(models, dict):
|
||||
model_list = models.get("data", [models]) if "data" in models else [models]
|
||||
for m in model_list:
|
||||
if not isinstance(m, dict):
|
||||
continue
|
||||
model_id = m.get("id", "unknown")
|
||||
_set_metric("llama_models_loaded", {"server": target["name"], "model": model_id}, 1.0)
|
||||
ctx_max = m.get("context_size", m.get("n_ctx", 0))
|
||||
if ctx_max:
|
||||
_set_metric("llama_model_context_max", {"server": target["name"], "model": model_id}, int(ctx_max))
|
||||
trained = m.get("train_tokens", m.get("n_train", 0))
|
||||
if trained:
|
||||
_set_metric("llama_model_tokens_trained", {"server": target["name"], "model": model_id}, int(trained))
|
||||
|
||||
return metrics
|
||||
|
||||
|
||||
|
||||
class MetricsHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
if self.path != "/metrics":
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
|
||||
all_metrics = {}
|
||||
|
||||
# Scrape llama.cpp targets
|
||||
targets = []
|
||||
env_targets = os.environ.get("LLAMA_TARGETS", "")
|
||||
if env_targets:
|
||||
try:
|
||||
targets = json.loads(env_targets)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("Invalid LLAMA_TARGETS JSON, using defaults")
|
||||
targets = []
|
||||
|
||||
if not targets:
|
||||
targets = LLAMA_CPP_DEFAULTS
|
||||
|
||||
for target in targets:
|
||||
if "url" not in target or "name" not in target:
|
||||
target["name"] = target.get("name", "default")
|
||||
target["url"] = target.get("url", "http://localhost:8012")
|
||||
try:
|
||||
all_metrics.update(_collect_llama_cpp(target))
|
||||
except Exception as e:
|
||||
logger.error("Error scraping llama.cpp target %s: %s", target.get("name", "unknown"), e)
|
||||
|
||||
lines = []
|
||||
for (metric_type, label_pairs), value in sorted(all_metrics.items()):
|
||||
labels = dict(label_pairs)
|
||||
if not labels:
|
||||
line = f"{metric_type} {value}"
|
||||
else:
|
||||
label_str = ",".join(f'{k}="{v}"' for k, v in sorted(labels.items()))
|
||||
line = f"{metric_type}{{{label_str}}} {value}"
|
||||
lines.append(line)
|
||||
|
||||
body = "\n".join(lines) + "\n" if lines else "# no metrics available\n"
|
||||
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
self.end_headers()
|
||||
self.wfile.write(body.encode("utf-8"))
|
||||
|
||||
def log_message(self, format, *args):
|
||||
logger.debug("%s - - %s", self.address_string(), format % args)
|
||||
|
||||
|
||||
def main():
|
||||
port = int(os.environ.get("LLAMA_EXPORTER_PORT", "9550"))
|
||||
host = os.environ.get("LLAMA_EXPORTER_BIND", "0.0.0.0")
|
||||
|
||||
server = HTTPServer((host, port), MetricsHandler)
|
||||
logger.info("Starting Llama Exporter on %s:%d", host, port)
|
||||
logger.info("Polling every %d seconds, timeout %ds", POLL_INTERVAL, SCRAPE_TIMEOUT)
|
||||
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Shutting down")
|
||||
server.shutdown()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user