Add GPU monitoring services and configurations

- Create systemd service templates for llama_exporter, nvidia_exporter, and podman.
- Add Prometheus configuration template for GPU metrics scraping.
- Introduce variables for GPU monitoring role in main.yml.
- Implement tasks for syncing llama models, including user setup and package installation.
- Update podman tasks to ensure proper sudo access and lingering for the podman user.
- Modify inventory to include gpu_monitoring group.
- Add Terraform modules for deploying Vaultwarden, including ingress and service configurations.
- Create Grafana dashboard for real-time GPU monitoring metrics.
- Update secrets and environment files to include Vaultwarden admin token.
This commit is contained in:
2026-05-29 23:42:14 +02:00
parent e9e28a5ca8
commit 350650ecc2
47 changed files with 3727 additions and 49 deletions
+4 -3
View File
@@ -2,7 +2,8 @@ locals {
app_version = var.tag
monitoring_folder = "${var.persistent_folder}/grafana"
grafana_datasource_config = file("${path.module}/resources/default.yml")
grafana_dashboard_provider = file("${path.module}/resources/dashboard-provider.yml")
grafana_trivy_dashboard = file("${path.module}/resources/trivy-vulnerabilities-dashboard.json")
grafana_datasource_config = file("${path.module}/resources/default.yml")
grafana_dashboard_provider = file("${path.module}/resources/dashboard-provider.yml")
grafana_trivy_dashboard = file("${path.module}/resources/trivy-vulnerabilities-dashboard.json")
grafana_gpu_realtime_dashboard = file("${path.module}/resources/gpu-realtime-dashboard.json")
}
@@ -28,6 +28,7 @@ resource "kubernetes_config_map" "grafana_dashboards" {
data = {
"trivy-vulnerabilities-dashboard.json" = local.grafana_trivy_dashboard
"gpu-realtime-dashboard.json" = local.grafana_gpu_realtime_dashboard
}
}
@@ -0,0 +1,550 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": true,
"panels": [
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "11.0.0",
"targets": [
{
"editorMode": "code",
"expr": "up{job=~\"prometheus|nvidia-gpu|amdgpu_exporter|llama_exporter\"}",
"instant": true,
"legendFormat": "{{job}}",
"range": false,
"refId": "A"
}
],
"title": "Exporter Health",
"type": "stat"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 6
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.0.0",
"targets": [
{
"editorMode": "code",
"expr": "{__name__=~\"gpu_.*_gpu_util_percent|gpu_util_percent\"}",
"legendFormat": "{{__name__}} {{gpu}}",
"range": true,
"refId": "A"
}
],
"title": "GPU Utilization",
"type": "timeseries"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "celsius"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 6
},
"id": 3,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.0.0",
"targets": [
{
"editorMode": "code",
"expr": "{__name__=~\"gpu_.*_temperature_celsius|temperature_celsius\"}",
"legendFormat": "{{__name__}} {{gpu}}",
"range": true,
"refId": "A"
}
],
"title": "GPU Temperature",
"type": "timeseries"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "watt"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 14
},
"id": 4,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.0.0",
"targets": [
{
"editorMode": "code",
"expr": "{__name__=~\"gpu_.*_power_watts|power_watts\"}",
"legendFormat": "{{__name__}} {{gpu}}",
"range": true,
"refId": "A"
}
],
"title": "GPU Power Draw",
"type": "timeseries"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 14
},
"id": 5,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.0.0",
"targets": [
{
"editorMode": "code",
"expr": "{__name__=~\"gpu_.*_memory_used_percent\"}",
"legendFormat": "nvidia",
"range": true,
"refId": "A"
},
{
"editorMode": "code",
"expr": "100 * mem_used_bytes / clamp_min(mem_used_bytes + mem_avail_bytes, 1)",
"legendFormat": "amd {{gpu}}",
"range": true,
"refId": "B"
}
],
"title": "GPU Memory Usage",
"type": "timeseries"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 22
},
"id": 6,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.0.0",
"targets": [
{
"editorMode": "code",
"expr": "scrape_duration_seconds{job=~\"nvidia-gpu|amdgpu_exporter|llama_exporter|prometheus\"}",
"legendFormat": "{{job}}",
"range": true,
"refId": "A"
}
],
"title": "Scrape Duration",
"type": "timeseries"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 22
},
"id": 7,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.0.0",
"targets": [
{
"editorMode": "code",
"expr": "llama_server_health",
"legendFormat": "health {{server}}",
"range": true,
"refId": "A"
},
{
"editorMode": "code",
"expr": "llama_models_loaded",
"legendFormat": "models {{server}} {{model}}",
"range": true,
"refId": "B"
}
],
"title": "Llama Server Metrics",
"type": "timeseries"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 30
},
"id": 8,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.0.0",
"targets": [
{
"editorMode": "code",
"expr": "{__name__=~\"gpu_.*_pstate\"}",
"legendFormat": "{{__name__}}",
"range": true,
"refId": "A"
}
],
"title": "NVIDIA P-State",
"type": "timeseries"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 30
},
"id": 9,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.0.0",
"targets": [
{
"editorMode": "code",
"expr": "{__name__=~\"gpu_.*_pcie_link_gen_current\"}",
"legendFormat": "pcie gen",
"range": true,
"refId": "A"
},
{
"editorMode": "code",
"expr": "{__name__=~\"gpu_.*_pcie_link_width_current\"}",
"legendFormat": "pcie width",
"range": true,
"refId": "B"
}
],
"title": "NVIDIA PCIe Link",
"type": "timeseries"
},
{
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"max": 1,
"min": 0,
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 38
},
"id": 10,
"options": {
"legend": {
"calcs": [],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "11.0.0",
"targets": [
{
"editorMode": "code",
"expr": "{__name__=~\"gpu_.*_throttle_sw_power_cap\"}",
"legendFormat": "sw power cap",
"range": true,
"refId": "A"
},
{
"editorMode": "code",
"expr": "{__name__=~\"gpu_.*_throttle_hw_thermal_slowdown\"}",
"legendFormat": "hw thermal slowdown",
"range": true,
"refId": "B"
},
{
"editorMode": "code",
"expr": "{__name__=~\"gpu_.*_throttle_hw_power_brake_slowdown\"}",
"legendFormat": "hw power brake slowdown",
"range": true,
"refId": "C"
}
],
"title": "NVIDIA Throttle Flags",
"type": "timeseries"
}
],
"refresh": "5s",
"schemaVersion": 39,
"style": "dark",
"tags": [
"gpu",
"monitoring",
"realtime"
],
"templating": {
"list": [
{
"current": {
"selected": true,
"text": "Prometheus",
"value": "Prometheus"
},
"hide": 0,
"includeAll": false,
"label": "Datasource",
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "GPU Monitoring Realtime",
"uid": "gpu-monitoring-realtime",
"version": 1,
"weekStart": ""
}
@@ -0,0 +1,6 @@
locals {
app_folder = var.persistent_folder
app_version = var.tag
app_fqdn = var.fqdn
db_path = var.db_folder != "" ? var.db_folder : local.app_folder
}
@@ -0,0 +1,39 @@
resource "kubernetes_ingress_v1" "vaultwarden" {
metadata {
name = "ingress"
namespace = kubernetes_namespace.vaultwarden.metadata[0].name
annotations = {
"kubernetes.io/ingress.class" = "public"
"cert-manager.io/cluster-issuer" = var.issuer
"kubernetes.io/tls-acme" = "true"
"nginx.ingress.kubernetes.io/ssl-redirect" = "true"
}
}
spec {
dynamic "tls" {
for_each = var.tls_enabled ? [1] : []
content {
hosts = [var.fqdn]
secret_name = "vaultwarden-tls"
}
}
rule {
host = var.fqdn
http {
path {
backend {
service {
name = kubernetes_service.vaultwarden.metadata[0].name
port {
number = 80
}
}
}
}
}
}
}
}
+191
View File
@@ -0,0 +1,191 @@
resource "kubernetes_namespace" "vaultwarden" {
metadata {
name = "vaultwarden"
}
}
resource "kubernetes_service_account" "vaultwarden" {
metadata {
name = "vaultwarden-sa"
namespace = kubernetes_namespace.vaultwarden.metadata[0].name
}
automount_service_account_token = false
}
resource "kubernetes_config_map" "vaultwarden_config" {
metadata {
name = "vaultwarden-config"
namespace = kubernetes_namespace.vaultwarden.metadata[0].name
}
data = {
DOMAIN = "https://${local.app_fqdn}/"
SIGNUP = var.signup_enabled ? "true" : "false"
DATABASE_URL = "file:///data/db.sqlite3"
}
}
resource "kubernetes_secret" "vaultwarden_secrets" {
metadata {
name = "vaultwarden-keys"
namespace = kubernetes_namespace.vaultwarden.metadata[0].name
}
data = {
ADMIN_TOKEN = var.admin_token
}
}
resource "kubernetes_deployment" "vaultwarden" {
metadata {
name = "vaultwarden"
namespace = kubernetes_namespace.vaultwarden.metadata[0].name
labels = {
app = "vaultwarden"
}
}
spec {
replicas = 1
selector {
match_labels = {
app = "vaultwarden"
}
}
strategy {
type = "Recreate"
}
template {
metadata {
labels = {
app = "vaultwarden"
}
}
spec {
service_account_name = kubernetes_service_account.vaultwarden.metadata[0].name
automount_service_account_token = false
security_context {
run_as_non_root = true
run_as_user = 1000
run_as_group = 1000
fs_group = 1000
seccomp_profile {
type = "RuntimeDefault"
}
}
container {
name = "vaultwarden"
image = "vaultwarden/server:${local.app_version}"
env_from {
config_map_ref {
name = kubernetes_config_map.vaultwarden_config.metadata[0].name
}
}
env {
name = "ADMIN_TOKEN"
value_from {
secret_key_ref {
name = kubernetes_secret.vaultwarden_secrets.metadata[0].name
key = "ADMIN_TOKEN"
}
}
}
security_context {
allow_privilege_escalation = false
privileged = false
capabilities {
drop = ["ALL"]
}
read_only_root_filesystem = true
}
port {
name = "http"
container_port = 80
protocol = "TCP"
}
liveness_probe {
http_get {
path = "/"
port = "http"
}
initial_delay_seconds = 30
period_seconds = 15
}
readiness_probe {
http_get {
path = "/"
port = "http"
}
initial_delay_seconds = 10
period_seconds = 10
}
volume_mount {
name = "vaultwarden-data"
mount_path = "/data"
}
volume_mount {
name = "vaultwarden-tmp"
mount_path = "/tmp"
}
}
volume {
name = "vaultwarden-data"
host_path {
path = local.db_path
}
}
volume {
name = "vaultwarden-tmp"
empty_dir {}
}
}
}
}
}
resource "kubernetes_service" "vaultwarden" {
metadata {
name = "vaultwarden-service"
namespace = kubernetes_namespace.vaultwarden.metadata[0].name
}
spec {
selector = {
app = "vaultwarden"
}
port {
name = "http"
port = 80
target_port = 80
}
type = "ClusterIP"
}
lifecycle {
ignore_changes = [
metadata[0].annotations
]
}
}
@@ -0,0 +1,9 @@
terraform {
required_version = "~>1.8"
required_providers {
kubernetes = {
source = "hashicorp/kubernetes"
version = "2.36.0"
}
}
}
@@ -0,0 +1,45 @@
variable "persistent_folder" {
description = "Path for persistent data on the host"
type = string
}
variable "fqdn" {
description = "FQDN for the service"
type = string
}
variable "tag" {
description = "Image tag to deploy"
type = string
default = "latest"
}
variable "issuer" {
description = "The cert-manager cluster issuer"
type = string
default = "letsencrypt-prod"
}
variable "tls_enabled" {
description = "Enable TLS in ingress"
type = bool
default = true
}
variable "signup_enabled" {
description = "Allow new user signups"
type = bool
default = false
}
variable "admin_token" {
description = "Admin token for VaultWarden"
type = string
sensitive = true
}
variable "db_folder" {
description = "Path for the database folder (default: persistent_folder)"
type = string
default = ""
}