Added auto-suspend, added wol proxy for ollama

This commit is contained in:
2025-06-22 18:43:55 +02:00
parent dab5520630
commit 6579e15394
16 changed files with 449 additions and 266 deletions
+41
View File
@@ -0,0 +1,41 @@
#!/bin/bash
IDLE_TIMEOUT_MINUTES=20
# Check 1: Any logged-in users?
if who | grep -qv "tty"; then
echo "Active user sessions found. Skipping suspend."
exit 0
fi
# Check 2: CPU idle percentage
CPU_IDLE=$(mpstat 1 1 | tail -n 1 | awk '/:/ {print $12}' | sed 's/,/./g')
CPU_THRESHOLD=90
if (( $(echo "$CPU_IDLE < $CPU_THRESHOLD" | bc -l) )); then
echo "CPU too active. Skipping suspend."
exit 0
fi
if ! command -v nvidia-smi &> /dev/null; then
echo "nvidia-smi not found, skipping GPU check"
else
# Check 3: Is GPU idle?
GPU_UTIL=$(nvidia-smi --query-gpu=utilization.gpu --format=csv,noheader,nounits | head -n 1)
GPU_THRESHOLD=10
if (( GPU_UTIL > GPU_THRESHOLD )); then
echo "GPU is busy (utilization=${GPU_UTIL}%). Skipping suspend."
exit 0
fi
# Check 4: Is OLLAMA active?
OLLAMA_ACTIVE=$(nvidia-smi --query-compute-apps=name,used_memory --format=csv,noheader,nounits | grep ollama | wc -l)
if (( OLLAMA_ACTIVE > 0 )); then
echo "OLLMAMA is active. Skipping suspend."
exit 0
fi
fi
echo "System idle. Suspending..."
shutdown now
@@ -38,11 +38,13 @@ fw_allowed_ports:
- { rule: "allow", port: "16443", proto: "tcp", from: "10.19.4.0/24" } # Kubernetes API (local network)
- { rule: "allow", port: "139", proto: "tcp", from: "10.19.4.0/24" } # Samba (local network only)
- { rule: "allow", port: "445", proto: "tcp", from: "10.19.4.0/24" } # Samba (local network only)
- { rule: "allow", port: "11434", proto: "tcp", from: "10.19.4.0/24" } # Ollama (local network only)
- { rule: "allow", port: "443", proto: "tcp", from: "10.5.5.2/32" } # HTTPS (Contabo VPN)
- { rule: "allow", port: "53", proto: "udp", from: "10.19.4.0/24" } # DNS (local network)
- { rule: "allow", port: "53", proto: "udp", from: "10.5.5.2/32" } # DNS (Contabo VPN)
ufw_outgoing_traffic:
- 9 # For Wake-on-LAN
- 22 # SSH
- 53 # DNS
- 80 # HTTP
+7
View File
@@ -0,0 +1,7 @@
- name: Setup Auto Suspend
hosts: auto_suspend
tasks:
- name: Setup Auto Suspend
ansible.builtin.include_tasks:
file: tasks/auto_suspend.yml
+129
View File
@@ -0,0 +1,129 @@
- name: Install dependencies
become: true
ansible.builtin.package:
name: "{{ item }}"
state: present
loop:
- sysstat
- bc
- name: Enable sysstat service
become: true
ansible.builtin.systemd:
name: sysstat
enabled: true
state: started
- name: Read existing GRUB_CMDLINE_LINUX
ansible.builtin.shell: set -o pipefail && grep '^GRUB_CMDLINE_LINUX=' /etc/default/grub | cut -d= -f2- | tr -d '"'
register: grub_line
changed_when: false
failed_when: grub_line.stdout == ""
- name: Set fact for existing_cmdline
ansible.builtin.set_fact:
existing_cmdline: "{{ grub_line.stdout }}"
- name: Set fedora grub file locations
ansible.builtin.set_fact:
grub_config_path_bios: /boot/grub2/grub.cfg
when: ansible_distribution == "Fedora" and ansible_os_family == "RedHat"
- name: Show current GRUB_CMDLINE_LINUX
ansible.builtin.debug:
msg: "Current GRUB_CMDLINE_LINUX: {{ existing_cmdline }}"
- name: Updated GRUB_CMDLINE_LINUX
ansible.builtin.debug:
msg: "Current GRUB_CMDLINE_LINUX: {{ existing_cmdline | regex_replace(' *mem_sleep_default=\\S+', '') }} mem_sleep_default=deep"
- name: Ensure mem_sleep_default=deep is set in GRUB_CMDLINE_LINUX
become: true
vars:
grub_cmdline_linux_line: >-
GRUB_CMDLINE_LINUX="{{ existing_cmdline | regex_replace(' *mem_sleep_default=\\S+', '') }} mem_sleep_default=deep"
ansible.builtin.lineinfile:
path: /etc/default/grub
regexp: '^GRUB_CMDLINE_LINUX='
line: "{{ grub_cmdline_linux_line }}"
- name: Generate grub config for BIOS
become: true
changed_when: false
ansible.builtin.command: grub2-mkconfig -o {{ grub_config_path_bios }}
- name: Create idle check and suspend script
become: true
ansible.builtin.copy:
dest: /usr/local/bin/auto_suspend_script.sh
mode: '0755'
owner: root
group: root
content: "{{ lookup('file', 'scripts/auto_suspend.sh') }}"
- name: Create systemd service for auto suspend
become: true
ansible.builtin.copy:
dest: /etc/systemd/system/auto-suspend.service
content: |
[Unit]
Description=Check system idleness and suspend if idle
After=multi-user.target
[Service]
Type=oneshot
ExecStart=/usr/local/bin/auto_suspend_script.sh
owner: root
group: root
mode: '0644'
- name: Create sleep.conf.d folder if it doesn't exist
become: true
ansible.builtin.file:
path: /etc/systemd/sleep.conf.d
state: directory
owner: root
group: root
mode: '0755'
- name: Create sleep.conf.d for deep sleep
become: true
ansible.builtin.copy:
dest: /etc/systemd/sleep.conf.d/mem-deep.conf
content: |
[Sleep]
MemorySleepMode=deep
owner: root
group: root
mode: '0644'
- name: Create systemd timer for auto suspend
become: true
ansible.builtin.copy:
dest: /etc/systemd/system/auto-suspend.timer
content: |
[Unit]
Description=Run idle suspend check every 10 minutes
[Timer]
OnBootSec=10min
OnUnitActiveSec=10min
Unit=auto-suspend.service
[Install]
WantedBy=timers.target
owner: root
group: root
mode: '0644'
- name: Reload systemd daemon
become: true
ansible.builtin.systemd:
daemon_reload: true
- name: Enable and start auto-suspend timer
become: true
ansible.builtin.systemd:
name: auto-suspend.timer
enabled: true
state: started
+3
View File
@@ -92,3 +92,6 @@ gpu-01.lab.alexpires.me
[wol]
gpu-01.lab.alexpires.me
[auto_suspend]
gpu-01.lab.alexpires.me
+1
View File
@@ -3,6 +3,7 @@ module "open-webui" {
persistent_folder = local.persistent_folder
fqdn = local.open_webui_fqdn
ollama_proxy = local.ollama_proxy
}
module "samba" {
+9 -1
View File
@@ -45,7 +45,7 @@ locals {
{
name = "ollama"
type = "CNAME"
content = "gpu-01.${local.lab_domain}."
content = "dev-01.${local.lab_domain}."
},
{
name = "kokoro"
@@ -103,6 +103,14 @@ locals {
}
}
ollama_proxy = {
enabled = true
mac = "a0:8c:fd:e2:3c:fa"
ip = "10.19.4.106"
port = "11434"
scheme = "http"
}
scaleway_smtp_host = "smtp.tem.scw.cloud"
nextcloud_fqdn = "cloud.alexpires.me"
-4
View File
@@ -1,4 +0,0 @@
locals {
comfyui_version = "frontend${var.tag}"
}
-98
View File
@@ -1,98 +0,0 @@
resource "kubernetes_namespace" "comfyui" {
metadata {
name = "comfyui"
}
}
resource "kubernetes_service_account" "comfyui" {
metadata {
name = "comfyui-sa"
namespace = kubernetes_namespace.comfyui.metadata[0].name
}
automount_service_account_token = false
}
resource "kubernetes_deployment" "comfyui" {
metadata {
name = "comfyui"
namespace = kubernetes_namespace.comfyui.metadata[0].name
labels = {
comfyui = "comfyui"
}
}
spec {
replicas = 1
selector {
match_labels = {
comfyui = "comfyui"
}
}
strategy {
type = "RollingUpdate"
}
template {
metadata {
labels = {
comfyui = "comfyui"
}
}
spec {
service_account_name = kubernetes_service_account.comfyui.metadata[0].name
automount_service_account_token = false
container {
name = "comfyui"
image = "a13labs/comfyui:${local.comfyui_version}"
security_context {
allow_privilege_escalation = false
}
port {
name = "frontend"
container_port = 5173
protocol = "TCP"
}
env {
name = "DEV_SERVER_COMFYUI_URL"
value = coalesce(var.backend, "http://localhost:8188")
}
}
}
}
}
}
resource "kubernetes_service" "comfyui" {
metadata {
name = "frontend"
namespace = kubernetes_namespace.comfyui.metadata[0].name
}
spec {
selector = {
comfyui = "comfyui"
}
port {
name = "frontend"
port = 5173
target_port = 5173
}
}
lifecycle {
ignore_changes = [
metadata[0].annotations
]
}
}
@@ -1,9 +0,0 @@
terraform {
required_version = "~>1.8"
required_providers {
kubernetes = {
source = "hashicorp/kubernetes"
version = "2.36.0"
}
}
}
@@ -1,11 +0,0 @@
variable "tag" {
description = "The version of app to install"
type = string
default = "v0.0.1"
}
variable "backend" {
description = "The backend to use for ComfyUI"
type = string
default = ""
}
@@ -0,0 +1,34 @@
resource "kubernetes_ingress_v1" "openwebui" {
metadata {
name = "ingress"
namespace = kubernetes_namespace.openwebui.metadata[0].name
annotations = {
"kubernetes.io/ingress.class" = "public"
"cert-manager.io/cluster-issuer" = "letsencrypt-prod"
"kubernetes.io/tls-acme" = "true"
"nginx.ingress.kubernetes.io/ssl-redirect" = "true"
}
}
spec {
tls {
hosts = [var.fqdn]
secret_name = "openwebui-tls"
}
rule {
host = var.fqdn
http {
path {
backend {
service {
name = "http"
port {
number = 8080
}
}
}
}
}
}
}
}
-143
View File
@@ -3,146 +3,3 @@ resource "kubernetes_namespace" "openwebui" {
name = "openwebui"
}
}
resource "kubernetes_service_account" "openwebui" {
metadata {
name = "openwebui-sa"
namespace = kubernetes_namespace.openwebui.metadata[0].name
}
automount_service_account_token = false
}
resource "kubernetes_deployment" "openwebui" {
metadata {
name = "openwebui"
namespace = kubernetes_namespace.openwebui.metadata[0].name
labels = {
openwebui = "openwebui"
}
}
spec {
replicas = 1
selector {
match_labels = {
openwebui = "openwebui"
}
}
strategy {
type = "RollingUpdate"
}
template {
metadata {
labels = {
openwebui = "openwebui"
}
}
spec {
service_account_name = kubernetes_service_account.openwebui.metadata[0].name
automount_service_account_token = false
container {
name = "openwebui-container"
image = "ghcr.io/open-webui/open-webui:${var.tag}"
security_context {
allow_privilege_escalation = false
}
port {
name = "http"
container_port = 8080
}
readiness_probe {
tcp_socket {
port = 8080
}
initial_delay_seconds = 5
period_seconds = 10
failure_threshold = 10
}
volume_mount {
name = "openwebui-data"
mount_path = "/app/backend/data"
}
}
volume {
name = "openwebui-data"
host_path {
path = local.openwebui_data
}
}
}
}
}
}
resource "kubernetes_service" "openwebui" {
metadata {
name = "http"
namespace = kubernetes_namespace.openwebui.metadata[0].name
}
spec {
selector = {
openwebui = "openwebui"
}
port {
name = "http"
port = 8080
target_port = 8080
}
cluster_ip = "None"
}
lifecycle {
ignore_changes = [
metadata[0].annotations
]
}
}
resource "kubernetes_ingress_v1" "openwebui" {
metadata {
name = "ingress"
namespace = kubernetes_namespace.openwebui.metadata[0].name
annotations = {
"kubernetes.io/ingress.class" = "public"
"cert-manager.io/cluster-issuer" = "letsencrypt-prod"
"kubernetes.io/tls-acme" = "true"
"nginx.ingress.kubernetes.io/ssl-redirect" = "true"
}
}
spec {
tls {
hosts = [var.fqdn]
secret_name = "openwebui-tls"
}
rule {
host = var.fqdn
http {
path {
backend {
service {
name = "http"
port {
number = 8080
}
}
}
}
}
}
}
}
@@ -0,0 +1,101 @@
resource "kubernetes_service_account" "ollama_proxy" {
count = var.ollama_proxy.enabled ? 1 : 0
metadata {
name = "ollama-proxy-sa"
namespace = kubernetes_namespace.openwebui.metadata[0].name
}
automount_service_account_token = false
}
resource "kubernetes_deployment" "ollama_proxy" {
count = var.ollama_proxy.enabled ? 1 : 0
metadata {
name = "ollama-proxy"
namespace = kubernetes_namespace.openwebui.metadata[0].name
labels = {
app = "ollama-proxy"
}
}
spec {
replicas = 1
selector {
match_labels = {
app = "ollama-proxy"
}
}
strategy {
type = "Recreate"
}
template {
metadata {
labels = {
app = "ollama-proxy"
}
}
spec {
service_account_name = kubernetes_service_account.openwebui.metadata[0].name
automount_service_account_token = false
host_network = true
container {
name = "app"
image = "a13labs/wol_proxy:v0.4.0"
image_pull_policy = "Always"
security_context {
capabilities {
add = ["NET_RAW"]
}
}
port {
name = "http"
container_port = var.ollama_proxy.port
}
env {
name = "UPSTREAM_MAC"
value = var.ollama_proxy.mac
}
env {
name = "UPSTREAM_IP"
value = var.ollama_proxy.ip
}
env {
name = "UPSTREAM_PORT"
value = var.ollama_proxy.port
}
env {
name = "UPSTREAM_SCHEME"
value = var.ollama_proxy.scheme
}
env {
name = "PROXY_PORT"
value = var.ollama_proxy.port
}
readiness_probe {
tcp_socket {
port = 11434
}
initial_delay_seconds = 5
period_seconds = 10
failure_threshold = 10
}
}
}
}
}
}
@@ -20,3 +20,17 @@ variable "issuer" {
default = "internal-ca"
}
variable "ollama_proxy" {
description = "Configuration for the Ollama proxy"
type = object({
enabled = bool
mac = optional(string, "")
ip = optional(string, "")
port = optional(string, "11434")
scheme = optional(string, "http")
})
default = {
enabled = false
}
}
+108
View File
@@ -0,0 +1,108 @@
resource "kubernetes_service_account" "openwebui" {
metadata {
name = "openwebui-sa"
namespace = kubernetes_namespace.openwebui.metadata[0].name
}
automount_service_account_token = false
}
resource "kubernetes_deployment" "openwebui" {
metadata {
name = "openwebui"
namespace = kubernetes_namespace.openwebui.metadata[0].name
labels = {
app = "openwebui"
}
}
spec {
replicas = 1
selector {
match_labels = {
app = "openwebui"
}
}
strategy {
type = "RollingUpdate"
}
template {
metadata {
labels = {
app = "openwebui"
}
}
spec {
service_account_name = kubernetes_service_account.openwebui.metadata[0].name
automount_service_account_token = false
container {
name = "app"
image = "ghcr.io/open-webui/open-webui:${var.tag}"
security_context {
allow_privilege_escalation = false
}
port {
name = "http"
container_port = 8080
}
readiness_probe {
tcp_socket {
port = 8080
}
initial_delay_seconds = 5
period_seconds = 10
failure_threshold = 10
}
volume_mount {
name = "openwebui-data"
mount_path = "/app/backend/data"
}
}
volume {
name = "openwebui-data"
host_path {
path = local.openwebui_data
}
}
}
}
}
}
resource "kubernetes_service" "openwebui" {
metadata {
name = "http"
namespace = kubernetes_namespace.openwebui.metadata[0].name
}
spec {
selector = {
app = "openwebui"
}
port {
name = "http"
port = 8080
target_port = 8080
}
cluster_ip = "None"
}
lifecycle {
ignore_changes = [
metadata[0].annotations
]
}
}