Add auth exporter module: implement Kubernetes resources, exporter logic, and testing

ansible/host_vars/prod-01.alexpires.me.yml

@@ -82,7 +82,7 @@ geoip_ssh_allowed_networks:
 
 # fail2ban
 fail2ban_jails:
-  ssh:
+  sshd:
     enabled: true
     port: "{{ sshd_port }}"
     filter: "sshd"

terraform/apps/dev-01/apps.tf

@@ -66,3 +66,8 @@ module "grafana" {
 module "node_exporter" {
   source = "../../modules/utils/node-exporter"
 }
+
+module "auth_exporter" {
+  source            = "../../modules/utils/auth-exporter"
+  persistent_folder = "${local.persistent_folder}/monitoring"
+}

terraform/modules/utils/auth-exporter/config.tf

@@ -0,0 +1,15 @@
+locals {
+  app_version = var.tag
+  workdir     = "${var.persistent_folder}/auth-exporter"
+  commands = [
+    "python -m venv /tmp/venv",
+    ". /tmp/venv/bin/activate",
+    "pip install -r /exporter/requirements.txt",
+    "python /exporter/exporter.py --export-usernames"
+  ]
+  exporter_script  = file("${path.module}/resources/exporter.py")
+  requirements_txt = file("${path.module}/resources/requirements.txt")
+
+  exporter_checksum     = md5(local.exporter_script)
+  requirements_checksum = md5(local.requirements_txt)
+}

terraform/modules/utils/auth-exporter/main.tf

@@ -0,0 +1,121 @@
+
+resource "kubernetes_service_account" "auth_exporter" {
+  metadata {
+    name      = "auth-exporter-sa"
+    namespace = var.namespace
+  }
+}
+
+resource "kubernetes_config_map" "auth_exporter" {
+  metadata {
+    name      = "auth-exporter-config"
+    namespace = var.namespace
+  }
+  data = {
+    "exporter.py"      = local.exporter_script
+    "requirements.txt" = local.requirements_txt
+  }
+}
+
+resource "kubernetes_deployment" "auth_exporter" {
+  metadata {
+    name      = "auth-exporter"
+    namespace = var.namespace
+    labels = {
+      app = "auth-exporter"
+    }
+  }
+  spec {
+    replicas = 1
+    selector {
+      match_labels = {
+        app = "auth-exporter"
+      }
+    }
+    template {
+      metadata {
+        labels = {
+          app = "auth-exporter"
+        }
+        annotations = {
+          "checksum/exporter"     = local.exporter_checksum
+          "checksum/requirements" = local.requirements_checksum
+        }
+      }
+      spec {
+        service_account_name            = kubernetes_service_account.auth_exporter.metadata[0].name
+        automount_service_account_token = false
+        container {
+          name    = "auth-exporter"
+          image   = var.tag
+          command = ["sh", "-c", join(" && ", local.commands)]
+          port {
+            container_port = 9100
+            name           = "http-metrics"
+          }
+          volume_mount {
+            name       = "exporter-volume"
+            mount_path = "/exporter"
+            read_only  = true
+          }
+          volume_mount {
+            name       = "workdir"
+            mount_path = "/tmp"
+          }
+          volume_mount {
+            name       = "host-logs"
+            mount_path = "/var/log"
+            read_only  = true
+          }
+          security_context {
+            run_as_user               = 0
+            run_as_group              = 0
+            read_only_root_filesystem = true
+          }
+        }
+        volume {
+          name = "exporter-volume"
+          config_map {
+            name = kubernetes_config_map.auth_exporter.metadata[0].name
+          }
+        }
+        volume {
+          name = "workdir"
+          host_path {
+            path = local.workdir
+            type = "DirectoryOrCreate"
+          }
+        }
+        volume {
+          name = "host-logs"
+          host_path {
+            path = "/var/log"
+            type = "Directory"
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service" "auth_exporter" {
+  metadata {
+    name      = "auth-exporter"
+    namespace = var.namespace
+    labels = {
+      app = "auth-exporter"
+    }
+  }
+  spec {
+    selector = {
+      app = "auth-exporter"
+    }
+    port {
+      port        = 9100
+      target_port = 9100
+      protocol    = "TCP"
+      name        = "http-metrics"
+    }
+    type = "ClusterIP"
+  }
+}

terraform/modules/utils/auth-exporter/provider.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = "~>1.8"
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.36.0"
+    }
+  }
+}

terraform/modules/utils/auth-exporter/resources/exporter.py

@@ -0,0 +1,267 @@
+#!/usr/bin/env python3
+
+import os
+import signal
+import argparse
+import logging
+import re
+import time
+from threading import Thread
+from wsgiref.simple_server import make_server
+from prometheus_client import make_wsgi_app, Counter, Gauge
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+
+# Aggregated, labeled metric (safe: labels have small domain)
+auth_sessions = Counter(
+    'auth_sessions_total',
+    'Total auth sessions started',
+    ['source', 'method']
+)
+
+# Optional, user-labeled metric. Disabled by default because usernames are high-cardinality.
+auth_sessions_by_user = Counter(
+    'auth_sessions_by_user_total',
+    'Total auth sessions by username (high-cardinality — enable with --export-usernames)',
+    ['source', 'method', 'username']
+)
+
+# exporter health/observability
+parse_errors = Counter('auth_exporter_parse_errors_total', 'Total parse errors')
+uptime = Gauge('auth_exporter_uptime_seconds', 'Exporter uptime in seconds')
+user_label_dropped = Counter('auth_exporter_user_labels_dropped_total', 'Number of username labels dropped due to max-user-labels limit')
+
+# more tolerant username pattern (allows -, ., @, digits)
+USERNAME = r'[\w\-\.\@]+'
+IP = r'[\d\.]+|[\da-fA-F:]+'
+
+pam_ssh_session = re.compile(
+    rf'sshd\[\d+\]: Accepted (password|publickey) for ({USERNAME}) from ({IP}) port \d+ ssh2'
+)
+tty_pattern = re.compile(
+    rf'login\[\d+\]: pam_unix\(login:session\): session opened for user ({USERNAME}) by \(uid=\d+\)'
+)
+systemd_logind_session = re.compile(
+    rf'systemd-logind\[\d+\]: New session \d+ of user ({USERNAME})\.'
+)
+pam_systemd_session = re.compile(
+    rf'systemd: pam_unix\(systemd-user:session\): session opened for user ({USERNAME})(?:\(uid=\d+\))? by \(uid=\d+\)'
+)
+pam_sudo_session = re.compile(
+    rf'sudo: pam_unix\(sudo[^)]*:session\): session opened for user ({USERNAME})(?:\(uid=\d+\))? by (?:({USERNAME})|uid=\d+|\(uid=\d+\))'
+)
+
+# Track seen usernames to limit cardinality if username export is enabled
+_seen_usernames = set()
+
+class TailReader:
+    def __init__(self, path, start_at_end=True):
+        self.path = path
+        self.file = None
+        self.inode = None
+        self.start_at_end = start_at_end
+        self.open_file()
+
+    def open_file(self):
+        try:
+            self.file = open(self.path, 'r', errors='ignore')
+            stat = os.fstat(self.file.fileno())
+            self.inode = (stat.st_ino, stat.st_dev)
+            if self.start_at_end:
+                self.file.seek(0, os.SEEK_END)
+            logging.info("Opened %s (inode=%s)", self.path, self.inode)
+        except Exception as e:
+            logging.error("Failed to open %s: %s", self.path, e)
+            raise
+
+    def check_rotate(self):
+        try:
+            stat = os.stat(self.path)
+            current = (stat.st_ino, stat.st_dev)
+            if current != self.inode:
+                logging.info("Detected rotation/truncate for %s (old=%s new=%s)", self.path, self.inode, current)
+                self.file.close()
+                self.open_file()
+            else:
+                # handle copytruncate: file same inode but possibly truncated
+                try:
+                    pos = self.file.tell()
+                    if pos > stat.st_size:
+                        logging.info("Detected file truncated for %s (pos=%d size=%d), seeking to %d", self.path, pos, stat.st_size, stat.st_size)
+                        self.file.seek(stat.st_size, os.SEEK_SET)
+                except Exception as e:
+                    logging.error("Error checking for truncation: %s", e)
+        except FileNotFoundError:
+            # file temporarily missing (rotation); try reopen later
+            logging.warning("File %s not found while monitoring; will retry", self.path)
+            time.sleep(1)
+        except Exception as e:
+            logging.error("Error checking rotation: %s", e)
+
+    def follow_once(self):
+        # yield any new lines
+        while True:
+            line = self.file.readline()
+            if not line:
+                break
+            yield line
+
+def parse_line(line, export_usernames=False, max_user_labels=100):
+    try:
+        # ssh (captures method and username)
+        m = pam_ssh_session.search(line)
+        if m:
+            method = m.group(1)  # 'password' or 'publickey'
+            username = m.group(2)
+            auth_sessions.labels(source='ssh', method=method).inc()
+            if export_usernames:
+                if username not in _seen_usernames:
+                    if len(_seen_usernames) >= max_user_labels:
+                        user_label_dropped.inc()
+                    else:
+                        _seen_usernames.add(username)
+                if username in _seen_usernames:
+                    auth_sessions_by_user.labels(source='ssh', method=method, username=username).inc()
+            return 'ssh'
+
+        # tty
+        m = tty_pattern.search(line)
+        if m:
+            username = m.group(1)
+            auth_sessions.labels(source='tty', method='local').inc()
+            if export_usernames:
+                if username not in _seen_usernames:
+                    if len(_seen_usernames) >= max_user_labels:
+                        user_label_dropped.inc()
+                    else:
+                        _seen_usernames.add(username)
+                if username in _seen_usernames:
+                    auth_sessions_by_user.labels(source='tty', method='local', username=username).inc()
+            return 'tty'
+
+        # systemd-logind
+        m = systemd_logind_session.search(line)
+        if m:
+            username = m.group(1)
+            auth_sessions.labels(source='systemd-logind', method='pam').inc()
+            if export_usernames:
+                if username not in _seen_usernames:
+                    if len(_seen_usernames) >= max_user_labels:
+                        user_label_dropped.inc()
+                    else:
+                        _seen_usernames.add(username)
+                if username in _seen_usernames:
+                    auth_sessions_by_user.labels(source='systemd-logind', method='pam', username=username).inc()
+            return 'systemd-logind'
+
+        # systemd user session via pam
+        m = pam_systemd_session.search(line)
+        if m:
+            username = m.group(1)
+            auth_sessions.labels(source='systemd-user', method='pam').inc()
+            if export_usernames:
+                if username not in _seen_usernames:
+                    if len(_seen_usernames) >= max_user_labels:
+                        user_label_dropped.inc()
+                    else:
+                        _seen_usernames.add(username)
+                if username in _seen_usernames:
+                    auth_sessions_by_user.labels(source='systemd-user', method='pam', username=username).inc()
+            return 'systemd-user'
+
+        # sudo
+        m = pam_sudo_session.search(line)
+        if m:
+            # sudo regex captures the target username (group 1) or similar; normalize to 'sudo'
+            username = m.group(1)
+            auth_sessions.labels(source='sudo', method='sudo').inc()
+            if export_usernames:
+                if username not in _seen_usernames:
+                    if len(_seen_usernames) >= max_user_labels:
+                        user_label_dropped.inc()
+                    else:
+                        _seen_usernames.add(username)
+                if username in _seen_usernames:
+                    auth_sessions_by_user.labels(source='sudo', method='sudo', username=username).inc()
+            return 'sudo'
+
+    except Exception:
+        parse_errors.inc()
+        logging.exception("Error parsing line")
+    return None
+
+running = True
+start_time = time.time()
+httpd = None
+_server_thread = None
+
+def handle_signal(signum, frame):
+    global running, httpd
+    logging.info("Received signal %s, shutting down", signum)
+    running = False
+    try:
+        if httpd:
+            logging.info("Shutting down HTTP server")
+            httpd.shutdown()
+    except Exception:
+        logging.exception("Error shutting down HTTP server")
+
+def main():
+    global running
+    parser = argparse.ArgumentParser(description="Auth log prometheus exporter")
+    parser.add_argument('--logfile', default='/var/log/auth.log', help='Path to auth log')
+    parser.add_argument('--port', type=int, default=9100, help='Prometheus metrics port')
+    parser.add_argument('--start-at-beginning', action='store_true', help='Parse file from beginning on first run')
+    parser.add_argument('--interval', type=float, default=1.0, help='Poll interval seconds')
+    parser.add_argument('--export-usernames', action='store_true', help='Enable exporting username labels (DANGEROUS: may increase cardinality)')
+    parser.add_argument('--max-user-labels', type=int, default=100, help='Maximum distinct username labels to export when --export-usernames is enabled')
+    args = parser.parse_args()
+
+    # NOTE: reading /var/log/auth.log usually requires root
+    if not os.access(args.logfile, os.R_OK):
+        logging.warning("No read access to %s. Exporter may need to run as root or use systemd-journal.", args.logfile)
+
+    # Start Prometheus WSGI app in a daemon thread so it does not block process exit.
+    global httpd, _server_thread
+    app = make_wsgi_app()
+    httpd = make_server('', args.port, app)
+    _server_thread = Thread(target=httpd.serve_forever, name="prometheus-http")
+    _server_thread.daemon = True
+    _server_thread.start()
+    logging.info("Starting Prometheus session exporter on port %d...", args.port)
+
+    tail = TailReader(args.logfile, start_at_end=not args.start_at_beginning)
+
+    signal.signal(signal.SIGTERM, handle_signal)
+    signal.signal(signal.SIGINT, handle_signal)
+
+    try:
+        while running:
+            tail.check_rotate()
+            for line in tail.follow_once():
+                parse_line(line, export_usernames=args.export_usernames, max_user_labels=args.max_user_labels)
+            uptime.set(time.time() - start_time)
+            time.sleep(args.interval)
+    except Exception:
+        logging.exception("Unhandled error in main loop")
+    finally:
+        logging.info("Shutting down exporter, cleaning up resources")
+        try:
+            if tail.file:
+                tail.file.close()
+        except Exception:
+            logging.exception("Error closing tail file")
+        try:
+            if httpd:
+                httpd.shutdown()
+        except Exception:
+            logging.exception("Error shutting down HTTP server")
+        try:
+            if _server_thread:
+                _server_thread.join(timeout=2)
+        except Exception:
+            logging.exception("Error joining HTTP server thread")
+        logging.info("Exporter stopped")
+
+if __name__ == '__main__':
+    main()

terraform/modules/utils/auth-exporter/resources/exporter_test.py

@@ -0,0 +1,53 @@
+import pathlib
+
+from exporter import parse_line
+
+
+def test_parse_samples():
+    """Ensure representative log lines are parsed to the expected source."""
+    samples = [
+        ("Oct  2 10:00:00 host sshd[123]: Accepted password for alice from 192.0.2.1 port 12345 ssh2", 'ssh'),
+        ("Oct  2 10:00:00 host sshd[124]: Accepted publickey for bob from 2001:db8::1 port 54321 ssh2", 'ssh'),
+        ("Oct  2 10:00:00 host login[200]: pam_unix(login:session): session opened for user root by (uid=0)", 'tty'),
+        ("Oct  2 10:00:00 host systemd-logind[735]: New session 42 of user alice.", 'systemd-logind'),
+        ("Oct  2 10:00:00 host systemd: pam_unix(systemd-user:session): session opened for user alice(uid=1000) by (uid=0)", 'systemd-user'),
+        ("Oct  2 10:00:00 host sudo: pam_unix(sudo:session): session opened for user root by (uid=1000)", 'sudo'),
+    ]
+
+    for line, expected in samples:
+        assert parse_line(line, export_usernames=False) == expected
+
+
+def test_fixture_parsing_no_cron_misclassified():
+    """Parse the provided fixtures/auth.log and ensure cron session lines are not mis-classified as tty/systemd/sudo/ssh."""
+    fixtures = pathlib.Path(__file__).parent / 'fixtures' / 'auth.log'
+    assert fixtures.exists(), f"fixture file missing: {fixtures}"
+
+    with fixtures.open('r') as fh:
+        lines = fh.readlines()
+
+    # Find a cron session line in the fixture and ensure it does not parse as a known auth session
+    cron_line = None
+    for l in lines:
+        if 'pam_unix(cron:session): session opened for user' in l:
+            cron_line = l
+            break
+
+    assert cron_line is not None, "cron sample line missing in fixture"
+    assert parse_line(cron_line, export_usernames=False) is None
+
+
+def test_export_usernames_limited():
+    """When exporting usernames, the exporter should limit unique username labels to max_user_labels."""
+    # import private set to inspect size; tests run in same process so this is fine
+    from exporter import _seen_usernames
+
+    # clear any previous state
+    _seen_usernames.clear()
+
+    # feed 5 distinct usernames but allow only 3 labels
+    for i in range(5):
+        line = f"Oct 2 10:00:00 host sshd[123]: Accepted password for user{i} from 1.2.3.{i} port 123 ssh2"
+        parse_line(line, export_usernames=True, max_user_labels=3)
+
+    assert len(_seen_usernames) == 3

terraform/modules/utils/auth-exporter/resources/requirements.txt

@@ -0,0 +1 @@
+prometheus_client==0.23.1

terraform/modules/utils/auth-exporter/variables.tf

@@ -0,0 +1,16 @@
+variable "tag" {
+  description = "Python image tag"
+  type        = string
+  default     = "python:3.12-slim"
+}
+
+variable "namespace" {
+  description = "Kubernetes namespace to deploy the prometheus exporter"
+  type        = string
+  default     = "kube-system"
+}
+
+variable "persistent_folder" {
+  description = "The path to the persistent folder"
+  type        = string
+}

terraform/modules/utils/cert_checker/variables.tf

@@ -5,7 +5,7 @@ variable "tag" {
 }
 
 variable "namespace" {
-  description = "Kubernetes namespace to deploy the metrics server"
+  description = "Kubernetes namespace to deploy the cert checker"
   type        = string
 }