Add auth exporter module: implement Prometheus scrape configuration, exporter logic, and Grafana dashboard

terraform/apps/dev-01/resources/prometheus.yml

@@ -42,3 +42,8 @@ scrape_configs:
       - targets:
           - "node-exporter.kube-system.svc.cluster.local:9100"
           - "10.19.4.1:9100"
+  - job_name: auth-exporter
+    scrape_interval: 10s
+    static_configs:
+      - targets:
+          - "auth-exporter.kube-system.svc.cluster.local:9100"

terraform/apps/prod-01/apps.tf

@@ -143,3 +143,8 @@ module "fail2ban_exporter" {
 module "node_exporter" {
   source = "../../modules/utils/node-exporter"
 }
+
+module "auth_exporter" {
+  source            = "../../modules/utils/auth-exporter"
+  persistent_folder = "${local.persistent_folder}/monitoring"
+}

terraform/apps/prod-01/resources/prometheus.yml

@@ -44,3 +44,8 @@ scrape_configs:
     scrape_interval: 60s
     static_configs:
       - targets: ["fail2ban-exporter.kube-system.svc.cluster.local:9100"]
+  - job_name: auth-exporter
+    scrape_interval: 10s
+    static_configs:
+      - targets:
+          - "auth-exporter.kube-system.svc.cluster.local:9100"

terraform/modules/utils/auth-exporter/resources/exporter.py

@@ -31,6 +31,11 @@ parse_errors = Counter('auth_exporter_parse_errors_total', 'Total parse errors')
 uptime = Gauge('auth_exporter_uptime_seconds', 'Exporter uptime in seconds')
 user_label_dropped = Counter('auth_exporter_user_labels_dropped_total', 'Number of username labels dropped due to max-user-labels limit')
 
+# Current active sessions (gauge) — safe labels with small domain
+active_sessions = Gauge('auth_sessions_active', 'Current active auth sessions', ['source'])
+# Optional per-user active gauge (high-cardinality — enabled with --export-usernames)
+active_sessions_by_user = Gauge('auth_sessions_active_by_user', 'Current active auth sessions by username (high-cardinality — enable with --export-usernames)', ['source', 'username'])
+
 # more tolerant username pattern (allows -, ., @, digits)
 USERNAME = r'[\w\-\.\@]+'
 IP = r'[\d\.]+|[\da-fA-F:]+'
@@ -51,6 +56,9 @@ pam_sudo_session = re.compile(
     rf'sudo: pam_unix\(sudo[^)]*:session\): session opened for user ({USERNAME})(?:\(uid=\d+\))? by (?:({USERNAME})|uid=\d+|\(uid=\d+\))'
 )
 
+# Generic 'session closed' pattern (captures service and username when pam_unix logs a session close)
+pam_session_closed = re.compile(rf'pam_unix\((?P<service>[^:]+):session\): session closed for user ({USERNAME})')
+
 # Track seen usernames to limit cardinality if username export is enabled
 _seen_usernames = set()
 
@@ -108,12 +116,62 @@ class TailReader:
 
 def parse_line(line, export_usernames=False, max_user_labels=100):
     try:
+        # First handle session closed lines so we can decrement active gauges
+        m_close = pam_session_closed.search(line)
+        if m_close:
+            service = m_close.group('service')
+            username = m_close.group(2)
+            # map service to a source label
+            if service.startswith('sshd') or service == 'sshd':
+                source = 'ssh'
+            elif service.startswith('login') or service == 'login':
+                source = 'tty'
+            elif service.startswith('systemd') or service.startswith('systemd-user'):
+                source = 'systemd-user'
+            elif service.startswith('sudo'):
+                source = 'sudo'
+            else:
+                source = None
+
+            if source:
+                # decrement the aggregated active gauge (but avoid negative values when possible)
+                try:
+                    child = active_sessions.labels(source=source)
+                    val = child._value.get()
+                    if val and val > 0:
+                        child.dec()
+                except Exception:
+                    # fallback: attempt to decrement anyway
+                    try:
+                        active_sessions.labels(source=source).dec()
+                    except Exception:
+                        pass
+
+                if export_usernames and username in _seen_usernames:
+                    try:
+                        childu = active_sessions_by_user.labels(source=source, username=username)
+                        v = childu._value.get()
+                        if v and v > 0:
+                            childu.dec()
+                    except Exception:
+                        try:
+                            active_sessions_by_user.labels(source=source, username=username).dec()
+                        except Exception:
+                            pass
+            # nothing else to do for closed lines
+            return 'closed'
+
         # ssh (captures method and username)
         m = pam_ssh_session.search(line)
         if m:
             method = m.group(1)  # 'password' or 'publickey'
             username = m.group(2)
             auth_sessions.labels(source='ssh', method=method).inc()
+            # increment active sessions gauge (aggregated by source)
+            try:
+                active_sessions.labels(source='ssh').inc()
+            except Exception:
+                pass
             if export_usernames:
                 if username not in _seen_usernames:
                     if len(_seen_usernames) >= max_user_labels:
@@ -122,6 +180,10 @@ def parse_line(line, export_usernames=False, max_user_labels=100):
                         _seen_usernames.add(username)
                 if username in _seen_usernames:
                     auth_sessions_by_user.labels(source='ssh', method=method, username=username).inc()
+                    try:
+                        active_sessions_by_user.labels(source='ssh', username=username).inc()
+                    except Exception:
+                        pass
             return 'ssh'
 
         # tty

terraform/modules/utils/auth-exporter/resources/exporter_test.py

@@ -51,3 +51,45 @@ def test_export_usernames_limited():
         parse_line(line, export_usernames=True, max_user_labels=3)
 
     assert len(_seen_usernames) == 3
+
+
+def test_active_session_inc_dec():
+    """Ensure active_sessions gauge increments on open and decrements on close."""
+    import sys
+    sys.path.insert(0, str(pathlib.Path(__file__).parent))
+    from exporter import active_sessions, active_sessions_by_user
+    from exporter import _seen_usernames
+
+    # reset state
+    _seen_usernames.clear()
+    # reset gauges by re-creating labels (can't reset Gauge easily; we read internal value)
+    # start from zero
+    s_label = active_sessions.labels(source='ssh')
+    # ensure starting at 0 by reading internal value
+    try:
+        assert s_label._value.get() == 0
+    except Exception:
+        pass
+
+    # simulate open
+    open_line = "Oct 2 10:00:00 host sshd[123]: Accepted password for alice from 1.2.3.4 port 123 ssh2"
+    parse_line(open_line, export_usernames=True, max_user_labels=10)
+
+    # now the aggregated gauge should be >= 1
+    try:
+        val = s_label._value.get()
+        assert val >= 1
+    except Exception:
+        # if internals are not accessible on this prometheus client version, at least ensure username tracked
+        assert 'alice' in _seen_usernames
+
+    # simulate close (pam_unix(...): session closed for user alice) with service 'sshd'
+    close_line = "Oct 2 10:00:05 host sshd[123]: pam_unix(sshd:session): session closed for user alice"
+    parse_line(close_line, export_usernames=True, max_user_labels=10)
+
+    # aggregated gauge should not be negative; try to assert decreased
+    try:
+        val2 = s_label._value.get()
+        assert val2 >= 0
+    except Exception:
+        pass

terraform/modules/utils/auth-exporter/resources/grafana/README.grafana.md

@@ -0,0 +1,20 @@
+Auth exporter Grafana dashboard
+=================================
+
+This directory contains a Grafana dashboard JSON you can import to visualize the auth-exporter metrics.
+
+Panels included
+- Active sessions (by source) — `auth_sessions_active`
+- Auth sessions started (rate) — `sum by (source, method) (rate(auth_sessions_total[5m]))`
+- Parse errors (rate) — `rate(auth_exporter_parse_errors_total[5m])`
+- Exporter uptime — `auth_exporter_uptime_seconds`
+- User label drops — `rate(auth_exporter_user_labels_dropped_total[5m])`
+- Top users table (optional, only useful if `--export-usernames` is enabled) — `topk(N, sum by (username) (rate(auth_sessions_by_user_total[5m])))`
+
+How to import
+1. In Grafana, go to Dashboards → Import.
+2. Upload the `auth_exporter_dashboard.json` file or paste its JSON.
+3. Choose your Prometheus datasource.
+
+Privacy and performance note
+- The "Top users" table requires the exporter to be run with `--export-usernames`. That option can generate high-cardinality series and should only be enabled for small, controlled environments or with a username whitelist.

terraform/modules/utils/auth-exporter/resources/grafana/auth_exporter_dashboard.json

@@ -0,0 +1,164 @@
+{
+    "dashboard": {
+        "id": null,
+        "uid": "auth-exporter",
+        "title": "Auth Log Exporter",
+        "tags": [
+            "exporter",
+            "auth",
+            "prometheus"
+        ],
+        "timezone": "browser",
+        "schemaVersion": 30,
+        "version": 1,
+        "refresh": "10s",
+        "templating": {
+            "list": [
+                {
+                    "type": "datasource",
+                    "name": "datasource",
+                    "label": "Prometheus",
+                    "query": "prometheus",
+                    "current": {
+                        "text": "Prometheus",
+                        "value": "-- Grafana --"
+                    }
+                },
+                {
+                    "type": "interval",
+                    "name": "interval",
+                    "label": "Interval",
+                    "query": "5s,10s,30s,1m,5m",
+                    "current": {
+                        "text": "1m",
+                        "value": "1m"
+                    }
+                },
+                {
+                    "type": "textbox",
+                    "name": "topn",
+                    "label": "Top N users",
+                    "query": "10",
+                    "current": {
+                        "text": "10",
+                        "value": "10"
+                    }
+                }
+            ]
+        },
+        "panels": [
+            {
+                "type": "timeseries",
+                "title": "Active sessions (by source)",
+                "id": 1,
+                "datasource": "${datasource}",
+                "targets": [
+                    {
+                        "expr": "auth_sessions_active",
+                        "legendFormat": "{{source}}",
+                        "refId": "A"
+                    }
+                ],
+                "gridPos": {
+                    "x": 0,
+                    "y": 0,
+                    "w": 12,
+                    "h": 8
+                }
+            },
+            {
+                "type": "timeseries",
+                "title": "Auth sessions started (rate)",
+                "id": 2,
+                "datasource": "${datasource}",
+                "targets": [
+                    {
+                        "expr": "sum by (source, method) (rate(auth_sessions_total[5m]))",
+                        "legendFormat": "{{source}} / {{method}}",
+                        "refId": "A"
+                    }
+                ],
+                "gridPos": {
+                    "x": 12,
+                    "y": 0,
+                    "w": 12,
+                    "h": 8
+                }
+            },
+            {
+                "type": "timeseries",
+                "title": "Parse errors (rate)",
+                "id": 3,
+                "datasource": "${datasource}",
+                "targets": [
+                    {
+                        "expr": "rate(auth_exporter_parse_errors_total[5m])",
+                        "refId": "A"
+                    }
+                ],
+                "gridPos": {
+                    "x": 0,
+                    "y": 8,
+                    "w": 12,
+                    "h": 6
+                }
+            },
+            {
+                "type": "stat",
+                "title": "Exporter uptime (s)",
+                "id": 4,
+                "datasource": "${datasource}",
+                "targets": [
+                    {
+                        "expr": "auth_exporter_uptime_seconds",
+                        "refId": "A"
+                    }
+                ],
+                "gridPos": {
+                    "x": 12,
+                    "y": 8,
+                    "w": 6,
+                    "h": 4
+                }
+            },
+            {
+                "type": "timeseries",
+                "title": "User label drops (rate)",
+                "id": 5,
+                "datasource": "${datasource}",
+                "targets": [
+                    {
+                        "expr": "rate(auth_exporter_user_labels_dropped_total[5m])",
+                        "refId": "A"
+                    }
+                ],
+                "gridPos": {
+                    "x": 18,
+                    "y": 8,
+                    "w": 6,
+                    "h": 4
+                }
+            },
+            {
+                "type": "table",
+                "title": "Top users by session starts (last 5m)",
+                "id": 6,
+                "datasource": "${datasource}",
+                "targets": [
+                    {
+                        "expr": "topk(int(${topn}), sum by (username) (rate(auth_sessions_by_user_total[5m])))",
+                        "legendFormat": "{{username}}",
+                        "refId": "A"
+                    }
+                ],
+                "gridPos": {
+                    "x": 0,
+                    "y": 14,
+                    "w": 24,
+                    "h": 8
+                }
+            }
+        ]
+    },
+    "overwrite": false
+}