diff --git a/terraform/apps/dev-01/resources/prometheus.yml b/terraform/apps/dev-01/resources/prometheus.yml index 442c74c..76387f7 100644 --- a/terraform/apps/dev-01/resources/prometheus.yml +++ b/terraform/apps/dev-01/resources/prometheus.yml @@ -42,3 +42,8 @@ scrape_configs: - targets: - "node-exporter.kube-system.svc.cluster.local:9100" - "10.19.4.1:9100" + - job_name: auth-exporter + scrape_interval: 10s + static_configs: + - targets: + - "auth-exporter.kube-system.svc.cluster.local:9100" diff --git a/terraform/apps/prod-01/apps.tf b/terraform/apps/prod-01/apps.tf index 99bcf25..da2671f 100644 --- a/terraform/apps/prod-01/apps.tf +++ b/terraform/apps/prod-01/apps.tf @@ -143,3 +143,8 @@ module "fail2ban_exporter" { module "node_exporter" { source = "../../modules/utils/node-exporter" } + +module "auth_exporter" { + source = "../../modules/utils/auth-exporter" + persistent_folder = "${local.persistent_folder}/monitoring" +} diff --git a/terraform/apps/prod-01/resources/prometheus.yml b/terraform/apps/prod-01/resources/prometheus.yml index 2e4bda5..0833657 100644 --- a/terraform/apps/prod-01/resources/prometheus.yml +++ b/terraform/apps/prod-01/resources/prometheus.yml @@ -44,3 +44,8 @@ scrape_configs: scrape_interval: 60s static_configs: - targets: ["fail2ban-exporter.kube-system.svc.cluster.local:9100"] + - job_name: auth-exporter + scrape_interval: 10s + static_configs: + - targets: + - "auth-exporter.kube-system.svc.cluster.local:9100" diff --git a/terraform/modules/utils/auth-exporter/resources/exporter.py b/terraform/modules/utils/auth-exporter/resources/exporter.py index b81d8bf..7e9a128 100644 --- a/terraform/modules/utils/auth-exporter/resources/exporter.py +++ b/terraform/modules/utils/auth-exporter/resources/exporter.py @@ -31,6 +31,11 @@ parse_errors = Counter('auth_exporter_parse_errors_total', 'Total parse errors') uptime = Gauge('auth_exporter_uptime_seconds', 'Exporter uptime in seconds') user_label_dropped = Counter('auth_exporter_user_labels_dropped_total', 'Number of username labels dropped due to max-user-labels limit') +# Current active sessions (gauge) — safe labels with small domain +active_sessions = Gauge('auth_sessions_active', 'Current active auth sessions', ['source']) +# Optional per-user active gauge (high-cardinality — enabled with --export-usernames) +active_sessions_by_user = Gauge('auth_sessions_active_by_user', 'Current active auth sessions by username (high-cardinality — enable with --export-usernames)', ['source', 'username']) + # more tolerant username pattern (allows -, ., @, digits) USERNAME = r'[\w\-\.\@]+' IP = r'[\d\.]+|[\da-fA-F:]+' @@ -51,6 +56,9 @@ pam_sudo_session = re.compile( rf'sudo: pam_unix\(sudo[^)]*:session\): session opened for user ({USERNAME})(?:\(uid=\d+\))? by (?:({USERNAME})|uid=\d+|\(uid=\d+\))' ) +# Generic 'session closed' pattern (captures service and username when pam_unix logs a session close) +pam_session_closed = re.compile(rf'pam_unix\((?P[^:]+):session\): session closed for user ({USERNAME})') + # Track seen usernames to limit cardinality if username export is enabled _seen_usernames = set() @@ -108,12 +116,62 @@ class TailReader: def parse_line(line, export_usernames=False, max_user_labels=100): try: + # First handle session closed lines so we can decrement active gauges + m_close = pam_session_closed.search(line) + if m_close: + service = m_close.group('service') + username = m_close.group(2) + # map service to a source label + if service.startswith('sshd') or service == 'sshd': + source = 'ssh' + elif service.startswith('login') or service == 'login': + source = 'tty' + elif service.startswith('systemd') or service.startswith('systemd-user'): + source = 'systemd-user' + elif service.startswith('sudo'): + source = 'sudo' + else: + source = None + + if source: + # decrement the aggregated active gauge (but avoid negative values when possible) + try: + child = active_sessions.labels(source=source) + val = child._value.get() + if val and val > 0: + child.dec() + except Exception: + # fallback: attempt to decrement anyway + try: + active_sessions.labels(source=source).dec() + except Exception: + pass + + if export_usernames and username in _seen_usernames: + try: + childu = active_sessions_by_user.labels(source=source, username=username) + v = childu._value.get() + if v and v > 0: + childu.dec() + except Exception: + try: + active_sessions_by_user.labels(source=source, username=username).dec() + except Exception: + pass + # nothing else to do for closed lines + return 'closed' + # ssh (captures method and username) m = pam_ssh_session.search(line) if m: method = m.group(1) # 'password' or 'publickey' username = m.group(2) auth_sessions.labels(source='ssh', method=method).inc() + # increment active sessions gauge (aggregated by source) + try: + active_sessions.labels(source='ssh').inc() + except Exception: + pass if export_usernames: if username not in _seen_usernames: if len(_seen_usernames) >= max_user_labels: @@ -122,6 +180,10 @@ def parse_line(line, export_usernames=False, max_user_labels=100): _seen_usernames.add(username) if username in _seen_usernames: auth_sessions_by_user.labels(source='ssh', method=method, username=username).inc() + try: + active_sessions_by_user.labels(source='ssh', username=username).inc() + except Exception: + pass return 'ssh' # tty diff --git a/terraform/modules/utils/auth-exporter/resources/exporter_test.py b/terraform/modules/utils/auth-exporter/resources/exporter_test.py index aa692fd..75052ee 100644 --- a/terraform/modules/utils/auth-exporter/resources/exporter_test.py +++ b/terraform/modules/utils/auth-exporter/resources/exporter_test.py @@ -51,3 +51,45 @@ def test_export_usernames_limited(): parse_line(line, export_usernames=True, max_user_labels=3) assert len(_seen_usernames) == 3 + + +def test_active_session_inc_dec(): + """Ensure active_sessions gauge increments on open and decrements on close.""" + import sys + sys.path.insert(0, str(pathlib.Path(__file__).parent)) + from exporter import active_sessions, active_sessions_by_user + from exporter import _seen_usernames + + # reset state + _seen_usernames.clear() + # reset gauges by re-creating labels (can't reset Gauge easily; we read internal value) + # start from zero + s_label = active_sessions.labels(source='ssh') + # ensure starting at 0 by reading internal value + try: + assert s_label._value.get() == 0 + except Exception: + pass + + # simulate open + open_line = "Oct 2 10:00:00 host sshd[123]: Accepted password for alice from 1.2.3.4 port 123 ssh2" + parse_line(open_line, export_usernames=True, max_user_labels=10) + + # now the aggregated gauge should be >= 1 + try: + val = s_label._value.get() + assert val >= 1 + except Exception: + # if internals are not accessible on this prometheus client version, at least ensure username tracked + assert 'alice' in _seen_usernames + + # simulate close (pam_unix(...): session closed for user alice) with service 'sshd' + close_line = "Oct 2 10:00:05 host sshd[123]: pam_unix(sshd:session): session closed for user alice" + parse_line(close_line, export_usernames=True, max_user_labels=10) + + # aggregated gauge should not be negative; try to assert decreased + try: + val2 = s_label._value.get() + assert val2 >= 0 + except Exception: + pass diff --git a/terraform/modules/utils/auth-exporter/resources/grafana/README.grafana.md b/terraform/modules/utils/auth-exporter/resources/grafana/README.grafana.md new file mode 100644 index 0000000..6cb1928 --- /dev/null +++ b/terraform/modules/utils/auth-exporter/resources/grafana/README.grafana.md @@ -0,0 +1,20 @@ +Auth exporter Grafana dashboard +================================= + +This directory contains a Grafana dashboard JSON you can import to visualize the auth-exporter metrics. + +Panels included +- Active sessions (by source) — `auth_sessions_active` +- Auth sessions started (rate) — `sum by (source, method) (rate(auth_sessions_total[5m]))` +- Parse errors (rate) — `rate(auth_exporter_parse_errors_total[5m])` +- Exporter uptime — `auth_exporter_uptime_seconds` +- User label drops — `rate(auth_exporter_user_labels_dropped_total[5m])` +- Top users table (optional, only useful if `--export-usernames` is enabled) — `topk(N, sum by (username) (rate(auth_sessions_by_user_total[5m])))` + +How to import +1. In Grafana, go to Dashboards → Import. +2. Upload the `auth_exporter_dashboard.json` file or paste its JSON. +3. Choose your Prometheus datasource. + +Privacy and performance note +- The "Top users" table requires the exporter to be run with `--export-usernames`. That option can generate high-cardinality series and should only be enabled for small, controlled environments or with a username whitelist. diff --git a/terraform/modules/utils/auth-exporter/resources/grafana/auth_exporter_dashboard.json b/terraform/modules/utils/auth-exporter/resources/grafana/auth_exporter_dashboard.json new file mode 100644 index 0000000..ba18a93 --- /dev/null +++ b/terraform/modules/utils/auth-exporter/resources/grafana/auth_exporter_dashboard.json @@ -0,0 +1,164 @@ +{ + "dashboard": { + "id": null, + "uid": "auth-exporter", + "title": "Auth Log Exporter", + "tags": [ + "exporter", + "auth", + "prometheus" + ], + "timezone": "browser", + "schemaVersion": 30, + "version": 1, + "refresh": "10s", + "templating": { + "list": [ + { + "type": "datasource", + "name": "datasource", + "label": "Prometheus", + "query": "prometheus", + "current": { + "text": "Prometheus", + "value": "-- Grafana --" + } + }, + { + "type": "interval", + "name": "interval", + "label": "Interval", + "query": "5s,10s,30s,1m,5m", + "current": { + "text": "1m", + "value": "1m" + } + }, + { + "type": "textbox", + "name": "topn", + "label": "Top N users", + "query": "10", + "current": { + "text": "10", + "value": "10" + } + } + ] + }, + "panels": [ + { + "type": "timeseries", + "title": "Active sessions (by source)", + "id": 1, + "datasource": "${datasource}", + "targets": [ + { + "expr": "auth_sessions_active", + "legendFormat": "{{source}}", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 0, + "w": 12, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "Auth sessions started (rate)", + "id": 2, + "datasource": "${datasource}", + "targets": [ + { + "expr": "sum by (source, method) (rate(auth_sessions_total[5m]))", + "legendFormat": "{{source}} / {{method}}", + "refId": "A" + } + ], + "gridPos": { + "x": 12, + "y": 0, + "w": 12, + "h": 8 + } + }, + { + "type": "timeseries", + "title": "Parse errors (rate)", + "id": 3, + "datasource": "${datasource}", + "targets": [ + { + "expr": "rate(auth_exporter_parse_errors_total[5m])", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 8, + "w": 12, + "h": 6 + } + }, + { + "type": "stat", + "title": "Exporter uptime (s)", + "id": 4, + "datasource": "${datasource}", + "targets": [ + { + "expr": "auth_exporter_uptime_seconds", + "refId": "A" + } + ], + "gridPos": { + "x": 12, + "y": 8, + "w": 6, + "h": 4 + } + }, + { + "type": "timeseries", + "title": "User label drops (rate)", + "id": 5, + "datasource": "${datasource}", + "targets": [ + { + "expr": "rate(auth_exporter_user_labels_dropped_total[5m])", + "refId": "A" + } + ], + "gridPos": { + "x": 18, + "y": 8, + "w": 6, + "h": 4 + } + }, + { + "type": "table", + "title": "Top users by session starts (last 5m)", + "id": 6, + "datasource": "${datasource}", + "targets": [ + { + "expr": "topk(int(${topn}), sum by (username) (rate(auth_sessions_by_user_total[5m])))", + "legendFormat": "{{username}}", + "refId": "A" + } + ], + "gridPos": { + "x": 0, + "y": 14, + "w": 24, + "h": 8 + } + } + ] + }, + "overwrite": false +} \ No newline at end of file