diff --git a/ansible/tasks/windows_containers.yml b/ansible/tasks/windows_containers.yml new file mode 100644 index 0000000..216e9bc --- /dev/null +++ b/ansible/tasks/windows_containers.yml @@ -0,0 +1,85 @@ +# Variables you can override: +# enable_hyperv: true|false (default false) +# install_docker: true|false (default true) + +- name: Ensure Windows Containers feature is installed + ansible.windows.win_feature: + name: Containers + state: present + include_management_tools: yes + register: containers_feature + +- name: Ensure Hyper-V (optional, needed for some isolation modes) + ansible.windows.win_feature: + name: Hyper-V + state: present + include_management_tools: yes + register: hyperv_feature + when: enable_hyperv | default(false) + +- name: Reboot if required after feature installation + ansible.windows.win_reboot: + msg: "Rebooting after enabling Containers/Hyper-V features" + pre_reboot_delay: 15 + when: containers_feature.reboot_required or + (enable_hyperv | default(false) and hyperv_feature.reboot_required | default(false)) + +- name: Ensure NuGet provider is available (for PowerShell packages) + ansible.windows.win_powershell: + script: | + if (-not (Get-PackageProvider -Name NuGet -ErrorAction SilentlyContinue)) { + Install-PackageProvider -Name NuGet -MinimumVersion 2.8.5.201 -Force + Write-Output "installed" + } else { + Write-Output "present" + } + register: nuget_provider + changed_when: "'installed' in nuget_provider.stdout" + +- name: Ensure DockerMsftProvider module (package provider) is installed + ansible.windows.win_psmodule: + name: DockerMsftProvider + state: present + when: install_docker | default(true) + +- name: Install/Update Docker (Mirantis / Microsoft packaged) runtime + ansible.windows.win_powershell: + script: | + $pkg = Get-Package -Name docker -ErrorAction SilentlyContinue + if (-not $pkg) { + Install-Package -Name docker -ProviderName DockerMsftProvider -Force + Write-Output "installed" + } else { + Write-Output "present" + } + register: docker_pkg + changed_when: "'installed' in docker_pkg.stdout" + when: install_docker | default(true) + +- name: Ensure Docker service is enabled and running + ansible.windows.win_service: + name: docker + start_mode: auto + state: started + when: install_docker | default(true) + +- name: (Optional) Pull a test Windows base image + ansible.windows.win_powershell: + script: | + docker image inspect mcr.microsoft.com/windows/nanoserver:ltsc2022 2>$null + if ($LASTEXITCODE -ne 0) { + docker pull mcr.microsoft.com/windows/nanoserver:ltsc2022 + Write-Output "pulled" + } else { + Write-Output "present" + } + register: base_image + changed_when: "'pulled' in base_image.stdout" + when: install_docker | default(true) + +- name: Debug summary + ansible.builtin.debug: + msg: + containers_feature_changed: "{{ containers_feature.changed }}" + hyperv_feature_changed: "{{ (hyperv_feature.changed if (enable_hyperv | default(false)) else false) | default(false) }}" + docker_installed_changed: "{{ (docker_pkg.changed if (install_docker | default(true)) else false) | default(false) }}" diff --git a/bin/devolo_monitor.py b/bin/devolo_monitor.py new file mode 100644 index 0000000..f921fbe --- /dev/null +++ b/bin/devolo_monitor.py @@ -0,0 +1,65 @@ +import json +import websocket +import time +import requests + +# Step 1: Find Chrome DevTools WebSocket endpoint +targets = requests.get("http://localhost:9222/json").json() +ws_url = targets[0]["webSocketDebuggerUrl"] + +# Step 2: Connect to the DevTools Protocol +ws = websocket.create_connection(ws_url) +print(f"Connected to {ws_url}") + +# Helper: receive until we get the response for a given id +def recv_until_id(expected_id, timeout=10): + ws.settimeout(timeout) + while True: + msg = json.loads(ws.recv()) + # Log protocol events to help debugging + if "id" not in msg: + print(msg) + continue + if msg["id"] == expected_id: + return msg + # Optionally log out-of-order responses + print(msg) + +# Step 3: Enable runtime +ws.send(json.dumps({"id": 1, "method": "Runtime.enable"})) +recv_until_id(1) + +# Step 4: Navigate to the page and wait a bit +ws.send(json.dumps({"id": 2, "method": "Page.navigate", "params": {"url": "http://192.168.0.199/#/powerline"}})) +recv_until_id(2) +time.sleep(2) # give the page some time to render + +# Step 5: Evaluate JavaScript to extract device data, and return by value +script = """ +Array.from(document.querySelectorAll('tr[id^="plc-"]')).map(tr => ({ + id: tr.querySelector('[id^="plc-id-"]')?.innerText?.trim(), + mac: tr.querySelector('[id^="plc-mac-"]')?.innerText?.trim(), + tx: tr.querySelector('[id^="plc-tx-"]')?.innerText?.trim(), + rx: tr.querySelector('[id^="plc-rx-"]')?.innerText?.trim() +})) +""" +ws.send(json.dumps({ + "id": 3, + "method": "Runtime.evaluate", + "params": { + "expression": script, + "returnByValue": True, + "awaitPromise": True + } +})) +response = recv_until_id(3) +result = response.get("result", {}).get("result", {}).get("value") + +print("\nšŸ“” Device Table:") +if result: + for dev in result: + print(f"ID: {dev['id']:<10} MAC: {dev['mac']:<20} TX: {dev['tx']:<6} RX: {dev['rx']}") +else: + print("No devices found — maybe the table hasn't loaded yet?") + +ws.close() diff --git a/bin/webcat b/bin/webcat new file mode 100755 index 0000000..455777d --- /dev/null +++ b/bin/webcat @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +import sys +import websocket + +if len(sys.argv) < 2: + print("Usage: python wsclient.py ws://localhost:9222/devtools/page/XYZ") + sys.exit(1) + +url = sys.argv[1] +ws = websocket.WebSocket() +ws.connect(url) + +print(f"Connected to {url}") +try: + while True: + msg = input("> ") + if msg.lower() in {"exit", "quit"}: + break + ws.send(msg) + print(ws.recv()) +except KeyboardInterrupt: + pass +finally: + ws.close() diff --git a/terraform/apps/dev-01/apps.tf b/terraform/apps/dev-01/apps.tf index 2e67da4..455fe9a 100644 --- a/terraform/apps/dev-01/apps.tf +++ b/terraform/apps/dev-01/apps.tf @@ -10,7 +10,7 @@ module "open-webui" { module "samba" { source = "../../modules/apps/samba" - shared_folder = local.samba_shared_folder + shares = local.samba_shares } module "dns" { @@ -73,20 +73,3 @@ module "grafana" { fqdn = local.grafana_fqdn } -module "node_exporter" { - source = "../../modules/utils/node-exporter" -} - -module "auth_exporter" { - source = "../../modules/utils/auth-exporter" - persistent_folder = "${local.persistent_folder}/monitoring" -} - -module "disk_exporter" { - source = "../../modules/utils/disk-exporter" - persistent_folder = "${local.persistent_folder}/monitoring" - smartctl_static_version = local.smartctl_static_version - smartctl_sha256 = local.smartctl_sha256 - check_frequency = 600 - standby_mode = false -} diff --git a/terraform/apps/dev-01/config.tf b/terraform/apps/dev-01/config.tf index 29de895..3cb98c8 100644 --- a/terraform/apps/dev-01/config.tf +++ b/terraform/apps/dev-01/config.tf @@ -8,7 +8,18 @@ locals { internal_issuer = "internal-ca" # samba - samba_shared_folder = "/mnt/usb" + samba_shares = [ + { + name = "USB" + path = "/mnt/usb" + comment = "USB Drive" + }, + { + name = "Storage" + path = "/srv/storage" + comment = "Main Storage" + } + ] # DNS dns_nameservers = [ @@ -127,6 +138,18 @@ locals { upstream = "dns.dns.svc.cluster.local" protocol = "udp" } + "smb" = { + port = 445 + host_port = 445 + upstream = "smb.samba.svc.cluster.local" + protocol = "tcp" + } + "netbios" = { + port = 139 + host_port = 139 + upstream = "netbios.samba.svc.cluster.local" + protocol = "tcp" + } } ollama_proxy = { @@ -191,6 +214,17 @@ locals { telegram_message = file("${path.module}/resources/telegram.tpl") grafana_fqdn = "grafana.${local.lab_domain}" + + devolo_devices = { + devolo_620 = { + ip = "192.168.0.199" + model = "620" + } + # devolo_326 = { + # ip = "192.168.0.200" + # model = "326" + # } + } } data "terraform_remote_state" "scaleway" { diff --git a/terraform/apps/dev-01/exporters.tf b/terraform/apps/dev-01/exporters.tf new file mode 100644 index 0000000..4ec827d --- /dev/null +++ b/terraform/apps/dev-01/exporters.tf @@ -0,0 +1,25 @@ +module "node_exporter" { + source = "../../modules/utils/node-exporter" +} + +module "auth_exporter" { + source = "../../modules/utils/auth-exporter" + persistent_folder = "${local.persistent_folder}/monitoring" +} + +module "disk_exporter" { + source = "../../modules/utils/disk-exporter" + persistent_folder = "${local.persistent_folder}/monitoring" + smartctl_static_version = local.smartctl_static_version + smartctl_sha256 = local.smartctl_sha256 + check_frequency = 600 + standby_mode = false +} + +module "devolo_exporter" { + for_each = local.devolo_devices + source = "../../modules/utils/devolo-expporter" + devolo_ip = each.value.ip + devolo_model = each.value.model +} + diff --git a/terraform/apps/dev-01/resources/prometheus.yml b/terraform/apps/dev-01/resources/prometheus.yml index 24026c4..fd77be3 100644 --- a/terraform/apps/dev-01/resources/prometheus.yml +++ b/terraform/apps/dev-01/resources/prometheus.yml @@ -21,3 +21,9 @@ scrape_configs: - "disk-exporter.prometheus.svc.cluster.local:9100" - labels: cluster: "dev-01" + - job_name: devolo-exporter + scrape_interval: 10s + static_configs: + - targets: + - "devolo-exporter-620.prometheus.svc.cluster.local:9100" + # - "devolo-exporter-326.prometheus.svc.cluster.local:9100" diff --git a/terraform/modules/apps/mail/main.tf b/terraform/modules/apps/mail/main.tf index 06291be..ce034d6 100644 --- a/terraform/modules/apps/mail/main.tf +++ b/terraform/modules/apps/mail/main.tf @@ -56,7 +56,7 @@ resource "kubernetes_manifest" "mail_certificate" { } module "cert_checker" { - source = "../../utils/cert_checker" + source = "../../utils/cert-checker" namespace = kubernetes_namespace.mail.metadata[0].name persistent_folder = local.mail_path deployments = local.deployments diff --git a/terraform/modules/apps/samba/config.tf b/terraform/modules/apps/samba/config.tf index 0c23084..c4b315b 100644 --- a/terraform/modules/apps/samba/config.tf +++ b/terraform/modules/apps/samba/config.tf @@ -1,4 +1,9 @@ locals { samba_version = var.tag + + samba_config = templatefile("${path.module}/resources/samba.conf.tpl", { + shares = var.shares + }) + samba_config_hash = sha256(local.samba_config) } diff --git a/terraform/modules/apps/samba/main.tf b/terraform/modules/apps/samba/main.tf index f38d046..f2a957d 100644 --- a/terraform/modules/apps/samba/main.tf +++ b/terraform/modules/apps/samba/main.tf @@ -26,6 +26,18 @@ resource "kubernetes_secret" "samba_secrets" { } } +resource "kubernetes_config_map" "samba_config" { + metadata { + name = "samba-config" + namespace = kubernetes_namespace.samba.metadata[0].name + } + + data = { + "smb.conf" = local.samba_config + } + +} + resource "kubernetes_deployment" "samba" { metadata { name = "samba" @@ -45,7 +57,7 @@ resource "kubernetes_deployment" "samba" { } strategy { - type = "Recreate" + type = "RollingUpdate" } template { @@ -71,14 +83,14 @@ resource "kubernetes_deployment" "samba" { port { name = "smb-1" container_port = 445 - host_port = 445 + host_port = var.host_port_enabled ? 445 : null protocol = "TCP" } port { name = "smb-2" container_port = 139 - host_port = 139 + host_port = var.host_port_enabled ? 139 : null protocol = "TCP" } @@ -103,15 +115,39 @@ resource "kubernetes_deployment" "samba" { } volume_mount { - name = "storage" - mount_path = "/storage" + name = "samba-config" + mount_path = "/etc/samba/smb.conf" + sub_path = "smb.conf" + read_only = true + } + + dynamic "volume_mount" { + for_each = var.shares + content { + name = lower("samba-share-${volume_mount.value.name}") + mount_path = "/host/${volume_mount.value.path}" + } } } volume { - name = "storage" - host_path { - path = var.shared_folder + name = "samba-config" + config_map { + name = kubernetes_config_map.samba_config.metadata[0].name + items { + key = "smb.conf" + path = "smb.conf" + } + } + } + dynamic "volume" { + for_each = var.shares + content { + name = lower("samba-share-${volume.value.name}") + host_path { + path = volume.value.path + type = "DirectoryOrCreate" + } } } } @@ -123,9 +159,9 @@ resource "kubernetes_deployment" "samba" { } } -resource "kubernetes_service" "samba" { +resource "kubernetes_service" "smb" { metadata { - name = "samba-ports" + name = "smb" namespace = kubernetes_namespace.samba.metadata[0].name } @@ -140,7 +176,7 @@ resource "kubernetes_service" "samba" { target_port = 445 } - type = "LoadBalancer" + type = "ClusterIP" } lifecycle { @@ -149,3 +185,31 @@ resource "kubernetes_service" "samba" { ] } } + +resource "kubernetes_service" "netbios" { + metadata { + name = "netbios" + namespace = kubernetes_namespace.samba.metadata[0].name + } + + spec { + selector = { + samba = "samba" + } + + port { + name = "netbios" + port = 139 + target_port = 139 + } + + type = "ClusterIP" + } + + lifecycle { + ignore_changes = [ + metadata[0].annotations + ] + } + +} diff --git a/terraform/modules/apps/samba/resources/samba.conf.tpl b/terraform/modules/apps/samba/resources/samba.conf.tpl new file mode 100644 index 0000000..5961c60 --- /dev/null +++ b/terraform/modules/apps/samba/resources/samba.conf.tpl @@ -0,0 +1,23 @@ +[global] + server string = samba + idmap config * : range = 3000-7999 + security = user + server min protocol = SMB2 + + # disable printing services + load printers = no + printing = bsd + printcap name = /dev/null + disable spoolss = yes + +%{for share in shares ~} +[${share.name}] + path = /host/${share.path} + comment = ${share.comment} + valid users = ${join(", ", share.valid_users)} + browseable = ${share.browseable} + writable = ${share.writable} + read only = ${share.read_only} + force user = ${share.force_user} + force group = ${share.force_group} +%{endfor ~} diff --git a/terraform/modules/apps/samba/variables.tf b/terraform/modules/apps/samba/variables.tf index ef44149..1a1096b 100644 --- a/terraform/modules/apps/samba/variables.tf +++ b/terraform/modules/apps/samba/variables.tf @@ -4,9 +4,19 @@ variable "tag" { default = "latest" } -variable "shared_folder" { +variable "shares" { description = "Path to the shared folder on the host" - type = string + type = list(object({ + name = string + path = string + comment = string + valid_users = optional(list(string), ["@smb"]) + browseable = optional(bool, true) + writable = optional(bool, true) + read_only = optional(bool, false) + force_user = optional(string, "root") + force_group = optional(string, "root") + })) } variable "samba_username" { @@ -21,3 +31,9 @@ variable "samba_password" { default = "samba" sensitive = true } + +variable "host_port_enabled" { + description = "Enable hostPort for SMB ports (445 and 139). Useful for direct access on the node IP." + type = bool + default = false +} diff --git a/terraform/modules/utils/cert_checker/config.tf b/terraform/modules/utils/cert-checker/config.tf similarity index 100% rename from terraform/modules/utils/cert_checker/config.tf rename to terraform/modules/utils/cert-checker/config.tf diff --git a/terraform/modules/utils/cert_checker/main.tf b/terraform/modules/utils/cert-checker/main.tf similarity index 100% rename from terraform/modules/utils/cert_checker/main.tf rename to terraform/modules/utils/cert-checker/main.tf diff --git a/terraform/modules/utils/cert_checker/provider.tf b/terraform/modules/utils/cert-checker/provider.tf similarity index 100% rename from terraform/modules/utils/cert_checker/provider.tf rename to terraform/modules/utils/cert-checker/provider.tf diff --git a/terraform/modules/utils/cert_checker/resources/check.py b/terraform/modules/utils/cert-checker/resources/check.py similarity index 100% rename from terraform/modules/utils/cert_checker/resources/check.py rename to terraform/modules/utils/cert-checker/resources/check.py diff --git a/terraform/modules/utils/cert_checker/resources/requirements.txt b/terraform/modules/utils/cert-checker/resources/requirements.txt similarity index 100% rename from terraform/modules/utils/cert_checker/resources/requirements.txt rename to terraform/modules/utils/cert-checker/resources/requirements.txt diff --git a/terraform/modules/utils/cert_checker/variables.tf b/terraform/modules/utils/cert-checker/variables.tf similarity index 100% rename from terraform/modules/utils/cert_checker/variables.tf rename to terraform/modules/utils/cert-checker/variables.tf diff --git a/terraform/modules/utils/devolo-expporter/config.tf b/terraform/modules/utils/devolo-expporter/config.tf new file mode 100644 index 0000000..d4ecec4 --- /dev/null +++ b/terraform/modules/utils/devolo-expporter/config.tf @@ -0,0 +1,12 @@ +locals { + app_version = var.tag + commands = [ + "pip install --no-cache-dir -r /exporter/requirements.txt -t /tmp/python", + "PYTHONPATH=/tmp/python exec python /exporter/exporter.py" + ] + exporter_script = file("${path.module}/resources/exporter_${var.devolo_model}.py") + requirements_txt = file("${path.module}/resources/requirements.txt") + + exporter_checksum = md5(local.exporter_script) + requirements_checksum = md5(local.requirements_txt) +} diff --git a/terraform/modules/utils/devolo-expporter/main.tf b/terraform/modules/utils/devolo-expporter/main.tf new file mode 100644 index 0000000..25a7d5b --- /dev/null +++ b/terraform/modules/utils/devolo-expporter/main.tf @@ -0,0 +1,214 @@ +resource "kubernetes_service_account" "devolo_exporter" { + metadata { + name = "devolo-exporter-sa-${var.devolo_model}" + namespace = var.namespace + } +} + +resource "kubernetes_config_map" "devolo_exporter" { + metadata { + name = "devolo-exporter-config-${var.devolo_model}" + namespace = var.namespace + } + data = { + "exporter.py" = local.exporter_script + "requirements.txt" = local.requirements_txt + } +} + +resource "kubernetes_deployment" "devolo_exporter" { + metadata { + name = "devolo-exporter-${var.devolo_model}" + namespace = var.namespace + labels = { + app = "devolo-exporter-${var.devolo_model}" + } + } + spec { + replicas = 1 + selector { + match_labels = { + app = "devolo-exporter-${var.devolo_model}" + } + } + strategy { + type = "RollingUpdate" + } + template { + metadata { + labels = { + app = "devolo-exporter-${var.devolo_model}" + } + annotations = { + "checksum/exporter" = local.exporter_checksum + "checksum/requirements" = local.requirements_checksum + } + } + spec { + service_account_name = kubernetes_service_account.devolo_exporter.metadata[0].name + automount_service_account_token = false + + container { + name = "devolo-exporter" + image = "python:${var.tag}" + command = ["sh", "-c", join(" && ", local.commands)] + + env { + name = "EXPORTER_PORT" + value = tostring(var.exporter_port) + } + env { + name = "DEVOLO_IP" + value = var.devolo_ip + } + env { + name = "RD_PORT" + value = tostring(var.rd_port) + } + port { + container_port = var.exporter_port + name = "http-metrics" + } + + liveness_probe { + http_get { + path = "/healthz" + port = var.exporter_port + } + initial_delay_seconds = 10 + period_seconds = 10 + timeout_seconds = 5 + failure_threshold = 3 + } + + readiness_probe { + http_get { + path = "/healthz" + port = var.exporter_port + } + initial_delay_seconds = 5 + period_seconds = 10 + timeout_seconds = 5 + failure_threshold = 3 + } + + volume_mount { + name = "exporter-volume" + mount_path = "/exporter" + read_only = true + } + + volume_mount { + name = "tmp" + mount_path = "/tmp" + } + + security_context { + run_as_user = 0 + run_as_group = 0 + read_only_root_filesystem = true + } + } + + # Chromium sidecar providing the DevTools endpoint used by the exporter + container { + name = "chromium" + image = "nextools/chromium:latest" + command = ["/usr/bin/chromium-browser"] + args = [ + "--disable-background-networking", + "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", + "--disable-renderer-backgrounding", + "--disable-breakpad", + "--disable-client-side-phishing-detection", + "--disable-default-apps", + "--disable-dev-shm-usage", + "--disable-extensions", + "--disable-gpu", + "--disable-popup-blocking", + "--disable-prompt-on-repost", + "--disable-sync", + "--disable-translate", + "--headless", + "--hide-scrollbars", + "--ignore-certificate-errors", + "--ignore-certificate-errors-spki-list", + "--ignore-ssl-errors", + "--metrics-recording-only", + "--mute-audio", + "--no-first-run", + "--no-sandbox", + "--remote-debugging-address=0.0.0.0", + "--remote-debugging-port=${var.rd_port}", + "--remote-allow-origins=http://localhost:9222", + "--safebrowsing-disable-auto-update", + "--user-data-dir=/home/chromium", + "http://${var.devolo_ip}/#/overview" + ] + + # Optional: expose the DevTools port within the pod (not needed outside) + port { + container_port = var.rd_port + name = "devtools" + } + + volume_mount { + name = "chromium-data" + mount_path = "/home/chromium" + } + + volume_mount { + name = "tmp" + mount_path = "/tmp" + } + + security_context { + run_as_user = 0 + run_as_group = 0 + read_only_root_filesystem = true + } + } + + volume { + name = "exporter-volume" + config_map { + name = kubernetes_config_map.devolo_exporter.metadata[0].name + } + } + + # Writable /tmp for venv/site-packages installed under /tmp/python + volume { + name = "tmp" + empty_dir {} + } + volume { + name = "chromium-data" + empty_dir {} + } + } + } + } +} + +resource "kubernetes_service" "devolo_exporter" { + metadata { + name = "devolo-exporter-${var.devolo_model}" + namespace = var.namespace + labels = { + app = "devolo-exporter-${var.devolo_model}" + } + } + spec { + selector = { + app = "devolo-exporter-${var.devolo_model}" + } + port { + port = var.exporter_port + target_port = var.exporter_port + protocol = "TCP" + name = "http-metrics" + } + type = "ClusterIP" + } +} diff --git a/terraform/modules/utils/devolo-expporter/provider.tf b/terraform/modules/utils/devolo-expporter/provider.tf new file mode 100644 index 0000000..2eaa508 --- /dev/null +++ b/terraform/modules/utils/devolo-expporter/provider.tf @@ -0,0 +1,9 @@ +terraform { + required_version = "~>1.8" + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.36.0" + } + } +} diff --git a/terraform/modules/utils/devolo-expporter/resources/exporter_326.py b/terraform/modules/utils/devolo-expporter/resources/exporter_326.py new file mode 100644 index 0000000..82db303 --- /dev/null +++ b/terraform/modules/utils/devolo-expporter/resources/exporter_326.py @@ -0,0 +1,373 @@ +#!/usr/bin/env python3 +import json +import os +import signal +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler, HTTPServer +from socketserver import ThreadingMixIn + +import requests +import websocket +from prometheus_client import ( + CollectorRegistry, + Gauge, + Info, + CONTENT_TYPE_LATEST, + generate_latest, +) + + +# Configuration via environment +DEVOLO_IP = os.getenv("DEVOLO_IP") +RD_PORT = int(os.getenv("RD_PORT", "9222")) +EXPORTER_PORT = int(os.getenv("EXPORTER_PORT", "9100")) + +if not DEVOLO_IP: + print("DEVOLO_IP env var is required", file=sys.stderr) + sys.exit(1) + + +class DevoloScraper: + def __init__(self, devolo_ip: str, rd_port: int) -> None: + self.devolo_ip = devolo_ip + self.rd_port = rd_port + self.ws = None + self.msg_id = 0 + self.lock = threading.Lock() + + def _next_id(self) -> int: + with self.lock: + self.msg_id += 1 + return self.msg_id + + def _connect(self) -> None: + # Find Chrome DevTools WebSocket endpoint + targets = requests.get(f"http://localhost:{self.rd_port}/json", timeout=5).json() + if not targets: + raise RuntimeError("No DevTools targets found") + ws_url = targets[0]["webSocketDebuggerUrl"] + self.ws = websocket.create_connection(ws_url, timeout=10) + + # Enable Runtime and Page domains + self._send({"method": "Runtime.enable"}) + self._send({"method": "Page.enable"}) + + def _send(self, payload: dict) -> dict: + if self.ws is None: + self._connect() + msg_id = self._next_id() + payload = {"id": msg_id, **payload} + self.ws.send(json.dumps(payload)) + return self._recv_until_id(msg_id) + + def _recv_until_id(self, expected_id: int, timeout: int = 10) -> dict: + self.ws.settimeout(timeout) + while True: + msg = json.loads(self.ws.recv()) + if "id" not in msg: + continue + if msg["id"] == expected_id: + return msg + + def scrape_overview(self) -> dict: + try: + # Navigate to the Overview page (no hash for 326 model) + self._send( + { + "method": "Page.navigate", + "params": {"url": f"http://{self.devolo_ip}/overview"}, + } + ) + time.sleep(2) + + overview_js = """ +(() => { + const val = (id) => { + const el = document.getElementById(id); + return (el && (el.innerText || el.textContent) ? (el.innerText || el.textContent).trim() : ""); + }; + return { + name: val('device-name'), + serial: val('serial-number'), + firmware: val('firmware-version'), + mac: val('mac-address'), + uptime: val('system-uptime'), + }; +})() +""" + resp = self._send( + { + "method": "Runtime.evaluate", + "params": { + "expression": overview_js, + "returnByValue": True, + "awaitPromise": True, + }, + } + ) + value = resp.get("result", {}).get("result", {}).get("value") + return value if isinstance(value, dict) else {} + except Exception as e: # noqa: BLE001 + print(f"Overview scrape error: {e}", file=sys.stderr) + return {} + + def scrape_links(self) -> list[dict]: + try: + # Ensure we are on the overview page where the connections table lives + self._send( + { + "method": "Page.navigate", + "params": {"url": f"http://{self.devolo_ip}/overview"}, + } + ) + time.sleep(2) + + js = r""" +(() => { + const cells = Array.from(document.querySelectorAll('td[id^="devices-id-"]:not([id$="-label"]):not([id$="-this"])')); + const isDigits = (s) => /^[0-9]+$/.test(s); + return cells.map((el) => { + const idx = el.id.substring('devices-id-'.length); + if (!isDigits(idx)) return null; + const id = (el.textContent || '').replace(/\s+/g, ' ').trim(); + const mac = (document.getElementById(`devices-mac-${idx}`)?.textContent || '').trim(); + const tx = (document.getElementById(`devices-tx-${idx}`)?.textContent || '').trim(); + const rx = (document.getElementById(`devices-rx-${idx}`)?.textContent || '').trim(); + return { id, mac, tx, rx }; + }).filter(Boolean); +})() +""" + resp = self._send( + { + "method": "Runtime.evaluate", + "params": { + "expression": js, + "returnByValue": True, + "awaitPromise": True, + }, + } + ) + value = resp.get("result", {}).get("result", {}).get("value") + if isinstance(value, list): + out = [] + for it in value: + if not isinstance(it, dict): + continue + dev_id = (it.get("id") or "").replace("\xa0", " ").strip() + mac = (it.get("mac") or "").strip() + # Skip header or self row or empty rows + if ( + not dev_id + or "(this" in dev_id.lower() + or dev_id.lower() == "device" + or mac.lower() == "mac address" + ): + continue + out.append( + { + "id": dev_id, + "mac": mac, + "tx": _to_float(it.get("tx")), + "rx": _to_float(it.get("rx")), + } + ) + return out + return [] + except Exception as e: # noqa: BLE001 + print(f"Links scrape error: {e}", file=sys.stderr) + return [] + + def close(self) -> None: + if self.ws is not None: + try: + self.ws.close() + finally: + self.ws = None + + +def _to_float(v): + try: + if v is None: + return float("nan") + s = str(v).strip().replace(",", ".") + # Normalize non-breaking spaces and dashes + s = s.replace("\xa0", " ").replace("–", "-") + s = "".join(ch for ch in s if (ch.isdigit() or ch in ".-")) + if s == "" or s == "-" or s == "---": + return float("nan") + return float(s) + except Exception: # noqa: BLE001 + return float("nan") + + +def _parse_uptime_seconds(s: str) -> float: + try: + s = (s or "").strip() + if not s: + return float("nan") + # Format examples: "1 days, 03:49:56" or "03:49:56" + days = 0 + time_part = s + if "," in s: + day_part, time_part = [p.strip() for p in s.split(",", 1)] + if "day" in day_part: # matches 'day' or 'days' + try: + days = int(day_part.split()[0]) + except Exception: # noqa: BLE001 + days = 0 + h, m, sec = map(int, time_part.split(":")) + return float(days * 86400 + h * 3600 + m * 60 + sec) + except Exception: # noqa: BLE001 + return float("nan") + + +class MetricsHandler(BaseHTTPRequestHandler): + registry = CollectorRegistry() + dev_tx = Gauge( + "devolo_tx_mbps", + "Devolo powerline TX throughput in Mbps", + ["device_id", "mac"], + registry=registry, + ) + dev_rx = Gauge( + "devolo_rx_mbps", + "Devolo powerline RX throughput in Mbps", + ["device_id", "mac"], + registry=registry, + ) + device_info = Info( + "devolo_device_info", + "Devolo device information", + registry=registry, + ) + uptime_seconds = Gauge( + "devolo_uptime_seconds", + "Devolo device uptime in seconds", + registry=registry, + ) + + scraper: DevoloScraper | None = None + + def do_GET(self): # noqa: N802 + if self.path == "/healthz": + ok, msg = _health_check() + status = 200 if ok else 503 + body = ("ok" if ok else msg).encode() + self.send_response(status) + self.send_header("Content-Type", "text/plain; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + + if self.path != "/metrics": + self.send_response(404) + self.end_headers() + return + + if MetricsHandler.scraper is not None: + # Overview + ov = MetricsHandler.scraper.scrape_overview() + if ov: + MetricsHandler.device_info.info( + { + "name": ov.get("name", ""), + "serial": ov.get("serial", ""), + "firmware": ov.get("firmware", ""), + "mac": ov.get("mac", ""), + } + ) + MetricsHandler.uptime_seconds.set( + _parse_uptime_seconds(ov.get("uptime", "")) + ) + + # Links + links = MetricsHandler.scraper.scrape_links() + for row in links: + did = row.get("id", "") + mac = row.get("mac", "") + tx = row.get("tx") + rx = row.get("rx") + if tx is not None: + MetricsHandler.dev_tx.labels(device_id=did, mac=mac).set(tx) + if rx is not None: + MetricsHandler.dev_rx.labels(device_id=did, mac=mac).set(rx) + + output = generate_latest(MetricsHandler.registry) + self.send_response(200) + self.send_header("Content-Type", CONTENT_TYPE_LATEST) + self.send_header("Content-Length", str(len(output))) + self.end_headers() + self.wfile.write(output) + + +class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): + daemon_threads = True + + +def run_server(): + scraper = DevoloScraper(DEVOLO_IP, RD_PORT) + MetricsHandler.scraper = scraper + + httpd = ThreadingHTTPServer(("0.0.0.0", EXPORTER_PORT), MetricsHandler) + stop_event = threading.Event() + + def handle_term(signum, frame): # noqa: ANN001, ARG001 + print("Shutting down...") + try: + threading.Thread( + target=httpd.shutdown, name="httpd-shutdown", daemon=True + ).start() + except Exception: + pass + stop_event.set() + + signal.signal(signal.SIGTERM, handle_term) + signal.signal(signal.SIGINT, handle_term) + + print( + f"Exporter listening on :{EXPORTER_PORT}, scraping Devolo(326) at {DEVOLO_IP}, DevTools port {RD_PORT}" + ) + server_thread = threading.Thread( + target=httpd.serve_forever, name="httpd-serve", daemon=True + ) + server_thread.start() + + stop_event.wait() + server_thread.join(timeout=5) + try: + httpd.server_close() + except Exception: + pass + scraper.close() + + +def _health_check() -> tuple[bool, str]: + """Quick health check for liveness/readiness. + + Returns (ok, message). ok=False with a short message describing the first failure. + Criteria: + - Chromium DevTools endpoint reachable on localhost:RD_PORT + - Device HTTP reachable on http://DEVOLO_IP + """ + try: + r = requests.get(f"http://localhost:{RD_PORT}/json", timeout=2) + if r.status_code >= 400: + return False, f"devtools http {r.status_code}" + except Exception as e: # noqa: BLE001 + return False, f"devtools error: {e}" + + try: + r = requests.get(f"http://{DEVOLO_IP}", timeout=3) + if r.status_code >= 400: + return False, f"device http {r.status_code}" + except Exception as e: # noqa: BLE001 + return False, f"device error: {e}" + + return True, "ok" + + +if __name__ == "__main__": + run_server() diff --git a/terraform/modules/utils/devolo-expporter/resources/exporter_620.py b/terraform/modules/utils/devolo-expporter/resources/exporter_620.py new file mode 100644 index 0000000..a050756 --- /dev/null +++ b/terraform/modules/utils/devolo-expporter/resources/exporter_620.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python3 +import json +import os +import signal +import sys +import threading +import time +from prometheus_client import Info +from http.server import BaseHTTPRequestHandler, HTTPServer +from socketserver import ThreadingMixIn + +import requests +import websocket +from prometheus_client import CollectorRegistry, Gauge, generate_latest, CONTENT_TYPE_LATEST + + +# Configuration via environment +DEVOLO_IP = os.getenv("DEVOLO_IP") +RD_PORT = int(os.getenv("RD_PORT", "9222")) +EXPORTER_PORT = int(os.getenv("EXPORTER_PORT", "9100")) + +if not DEVOLO_IP: + print("DEVOLO_IP env var is required", file=sys.stderr) + sys.exit(1) + + +class DevoloScraper: + def __init__(self, devolo_ip: str, rd_port: int) -> None: + self.devolo_ip = devolo_ip + self.rd_port = rd_port + self.ws = None + self.msg_id = 0 + self.lock = threading.Lock() + + def _next_id(self) -> int: + with self.lock: + self.msg_id += 1 + return self.msg_id + + def _connect(self) -> None: + # Find Chrome DevTools WebSocket endpoint + targets = requests.get(f"http://localhost:{self.rd_port}/json", timeout=5).json() + if not targets: + raise RuntimeError("No DevTools targets found") + ws_url = targets[0]["webSocketDebuggerUrl"] + self.ws = websocket.create_connection(ws_url, timeout=10) + + # Enable Runtime domain + self._send({"method": "Runtime.enable"}) + # Enable Page domain for navigation + self._send({"method": "Page.enable"}) + + def _send(self, payload: dict) -> dict: + if self.ws is None: + self._connect() + msg_id = self._next_id() + payload = {"id": msg_id, **payload} + self.ws.send(json.dumps(payload)) + return self._recv_until_id(msg_id) + + def _recv_until_id(self, expected_id: int, timeout: int = 10) -> dict: + self.ws.settimeout(timeout) + while True: + msg = json.loads(self.ws.recv()) + if "id" not in msg: + # Log events but continue + continue + if msg["id"] == expected_id: + return msg + + def scrape(self) -> list[dict]: + try: + # Navigate to the Devolo page + self._send({ + "method": "Page.navigate", + "params": {"url": f"http://{self.devolo_ip}/#/powerline"}, + }) + time.sleep(2) + + # Evaluate JavaScript to extract device data and return by value + script = """ +Array.from(document.querySelectorAll('tr[id^="plc-"]')).map(tr => ({ + id: tr.querySelector('[id^="plc-id-"]')?.innerText?.trim(), + mac: tr.querySelector('[id^="plc-mac-"]')?.innerText?.trim(), + tx: tr.querySelector('[id^="plc-tx-"]')?.innerText?.trim(), + rx: tr.querySelector('[id^="plc-rx-"]')?.innerText?.trim() +})) +""" + resp = self._send({ + "method": "Runtime.evaluate", + "params": { + "expression": script, + "returnByValue": True, + "awaitPromise": True, + }, + }) + value = resp.get("result", {}).get("result", {}).get("value") + if isinstance(value, list): + # Normalize values + out = [] + for it in value: + if not isinstance(it, dict): + continue + dev_id = (it.get("id") or "").strip() + if not dev_id or "(this device)" in dev_id.lower(): + continue + out.append( + { + "id": (it.get("id") or "").strip(), + "mac": (it.get("mac") or "").strip(), + "tx": _to_float(it.get("tx")), + "rx": _to_float(it.get("rx")), + } + ) + return out + return [] + except Exception as e: # noqa: BLE001 + # Reset connection next time + try: + if self.ws is not None: + self.ws.close() + finally: + self.ws = None + print(f"Scrape error: {e}", file=sys.stderr) + return [] + + def scrape_overview(self) -> dict: + try: + # Navigate to the Overview page + self._send({ + "method": "Page.navigate", + "params": {"url": f"http://{self.devolo_ip}/#/overview"}, + }) + time.sleep(2) + + overview_js = """ +(() => { +const byId = (id) => { + const el = document.getElementById(id); + return (el && (el.innerText || el.textContent) ? (el.innerText || el.textContent).trim() : ""); +}; +return { + name: byId('system-board-name'), + serial: byId('system-serial-number'), + firmware: byId('system-firmware-version'), + mac: byId('brlan-macaddress'), + uptime: byId('system-timezone'), +}; +})() +""" + resp = self._send({ + "method": "Runtime.evaluate", + "params": { + "expression": overview_js, + "returnByValue": True, + "awaitPromise": True, + }, + }) + value = resp.get("result", {}).get("result", {}).get("value") + return value if isinstance(value, dict) else {} + except Exception as e: # noqa: BLE001 + print(f"Overview scrape error: {e}", file=sys.stderr) + return {} + + def close(self) -> None: + if self.ws is not None: + try: + self.ws.close() + finally: + self.ws = None + + +def _to_float(v): + try: + if v is None: + return float("nan") + s = str(v).strip().replace(",", ".") + if s == "---" or s == "": + return float("nan") + return float(s) + except Exception: # noqa: BLE001 + return float("nan") + + +def _parse_uptime_seconds(s: str) -> float: + try: + s = (s or "").strip() + if not s: + return float("nan") + # Format examples: "66 days, 16:51:36" or "16:51:36" + days = 0 + time_part = s + if "," in s: + day_part, time_part = [p.strip() for p in s.split(",", 1)] + if "day" in day_part: + try: + days = int(day_part.split()[0]) + except Exception: # noqa: BLE001 + days = 0 + hms = time_part.split(":") + if len(hms) != 3: + return float("nan") + h, m, sec = map(int, hms) + return float(days * 86400 + h * 3600 + m * 60 + sec) + except Exception: # noqa: BLE001 + return float("nan") + + + +def _health_check() -> tuple[bool, str]: + """Quick health check for liveness/readiness. + + Returns (ok, message). ok=False with a short message describing the first failure. + Criteria: + - Chromium DevTools endpoint reachable on localhost:RD_PORT + - Device HTTP reachable on http://DEVOLO_IP + """ + # 1) Check Chromium DevTools + try: + r = requests.get(f"http://localhost:{RD_PORT}/json", timeout=2) + if r.status_code >= 400: + return False, f"devtools http {r.status_code}" + except Exception as e: # noqa: BLE001 + return False, f"devtools error: {e}" + + # 2) Check device reachability + try: + r = requests.get(f"http://{DEVOLO_IP}", timeout=3) + if r.status_code >= 400: + return False, f"device http {r.status_code}" + except Exception as e: # noqa: BLE001 + return False, f"device error: {e}" + + return True, "ok" +class MetricsHandler(BaseHTTPRequestHandler): + registry = CollectorRegistry() + dev_tx = Gauge( + "devolo_tx_mbps", + "Devolo powerline TX throughput in Mbps", + ["device_id", "mac"], + registry=registry, + ) + dev_rx = Gauge( + "devolo_rx_mbps", + "Devolo powerline RX throughput in Mbps", + ["device_id", "mac"], + registry=registry, + ) + device_info = Info( + "devolo_device_info", + "Devolo device information", + registry=registry, + ) + uptime_seconds = Gauge( + "devolo_uptime_seconds", + "Devolo device uptime in seconds", + registry=registry, + ) + + scraper: DevoloScraper | None = None + + def do_GET(self): # noqa: N802 + if self.path == "/healthz": + ok, msg = _health_check() + status = 200 if ok else 503 + body = ("ok" if ok else msg).encode() + self.send_response(status) + self.send_header("Content-Type", "text/plain; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + + if self.path != "/metrics": + self.send_response(404) + self.end_headers() + return + + # Scrape fresh values on each scrape + if MetricsHandler.scraper is not None: + # Overview + ov = MetricsHandler.scraper.scrape_overview() + if ov: + MetricsHandler.device_info.info( + { + "name": ov.get("name", ""), + "serial": ov.get("serial", ""), + "firmware": ov.get("firmware", ""), + "mac": ov.get("mac", ""), + } + ) + MetricsHandler.uptime_seconds.set(_parse_uptime_seconds(ov.get("uptime", ""))) + + data = MetricsHandler.scraper.scrape() + # Clear previous values by resetting series to NaN, then set current + # Prometheus client does not provide a direct clear; we'll set current samples. + for row in data: + did = row.get("id", "") + mac = row.get("mac", "") + tx = row.get("tx") + rx = row.get("rx") + if tx is not None: + MetricsHandler.dev_tx.labels(device_id=did, mac=mac).set(tx) + if rx is not None: + MetricsHandler.dev_rx.labels(device_id=did, mac=mac).set(rx) + + output = generate_latest(MetricsHandler.registry) + self.send_response(200) + self.send_header("Content-Type", CONTENT_TYPE_LATEST) + self.send_header("Content-Length", str(len(output))) + self.end_headers() + self.wfile.write(output) + + +class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): + daemon_threads = True + +def run_server(): + scraper = DevoloScraper(DEVOLO_IP, RD_PORT) + MetricsHandler.scraper = scraper + + httpd = ThreadingHTTPServer(("0.0.0.0", EXPORTER_PORT), MetricsHandler) + stop_event = threading.Event() + + def handle_term(signum, frame): # noqa: ANN001, ARG001 + print("Shutting down...") + # Call shutdown from another thread to avoid deadlock with serve_forever loop + try: + threading.Thread(target=httpd.shutdown, name="httpd-shutdown", daemon=True).start() + except Exception: + pass + stop_event.set() + + signal.signal(signal.SIGTERM, handle_term) + signal.signal(signal.SIGINT, handle_term) + + print(f"Exporter listening on :{EXPORTER_PORT}, scraping Devolo at {DEVOLO_IP}, DevTools port {RD_PORT}") + # Run server in background so shutdown() is invoked from another thread safely + server_thread = threading.Thread(target=httpd.serve_forever, name="httpd-serve", daemon=True) + server_thread.start() + + # Wait until a termination signal triggers shutdown + stop_event.wait() + + # Give the server thread a brief moment to exit + server_thread.join(timeout=5) + try: + httpd.server_close() + except Exception: + pass + scraper.close() + +if __name__ == "__main__": + run_server() + diff --git a/terraform/modules/utils/devolo-expporter/resources/requirements.txt b/terraform/modules/utils/devolo-expporter/resources/requirements.txt new file mode 100644 index 0000000..b58726c --- /dev/null +++ b/terraform/modules/utils/devolo-expporter/resources/requirements.txt @@ -0,0 +1,3 @@ +prometheus_client==0.23.1 +requests==2.32.3 +websocket-client==1.8.0 \ No newline at end of file diff --git a/terraform/modules/utils/devolo-expporter/variables.tf b/terraform/modules/utils/devolo-expporter/variables.tf new file mode 100644 index 0000000..596b9fb --- /dev/null +++ b/terraform/modules/utils/devolo-expporter/variables.tf @@ -0,0 +1,33 @@ +variable "tag" { + description = "Python image tag" + type = string + default = "3.12-slim" +} + +variable "namespace" { + description = "Kubernetes namespace to deploy the prometheus exporter" + type = string + default = "prometheus" +} + +variable "exporter_port" { + description = "Port where the Prometheus exporter serves metrics" + type = number + default = 9100 +} + +variable "rd_port" { + description = "Chromium Remote Debugging port inside the pod" + type = number + default = 9222 +} + +variable "devolo_ip" { + description = "IP address of the Devolo device" + type = string +} + +variable "devolo_model" { + description = "Model of the Devolo device (e.g., '620' for Devolo Magic 2 WiFi next)" + type = string +}