Add auth exporter module: implement Prometheus scrape configuration, exporter logic, and Grafana dashboard
This commit is contained in:
@@ -31,6 +31,11 @@ parse_errors = Counter('auth_exporter_parse_errors_total', 'Total parse errors')
|
||||
uptime = Gauge('auth_exporter_uptime_seconds', 'Exporter uptime in seconds')
|
||||
user_label_dropped = Counter('auth_exporter_user_labels_dropped_total', 'Number of username labels dropped due to max-user-labels limit')
|
||||
|
||||
# Current active sessions (gauge) — safe labels with small domain
|
||||
active_sessions = Gauge('auth_sessions_active', 'Current active auth sessions', ['source'])
|
||||
# Optional per-user active gauge (high-cardinality — enabled with --export-usernames)
|
||||
active_sessions_by_user = Gauge('auth_sessions_active_by_user', 'Current active auth sessions by username (high-cardinality — enable with --export-usernames)', ['source', 'username'])
|
||||
|
||||
# more tolerant username pattern (allows -, ., @, digits)
|
||||
USERNAME = r'[\w\-\.\@]+'
|
||||
IP = r'[\d\.]+|[\da-fA-F:]+'
|
||||
@@ -51,6 +56,9 @@ pam_sudo_session = re.compile(
|
||||
rf'sudo: pam_unix\(sudo[^)]*:session\): session opened for user ({USERNAME})(?:\(uid=\d+\))? by (?:({USERNAME})|uid=\d+|\(uid=\d+\))'
|
||||
)
|
||||
|
||||
# Generic 'session closed' pattern (captures service and username when pam_unix logs a session close)
|
||||
pam_session_closed = re.compile(rf'pam_unix\((?P<service>[^:]+):session\): session closed for user ({USERNAME})')
|
||||
|
||||
# Track seen usernames to limit cardinality if username export is enabled
|
||||
_seen_usernames = set()
|
||||
|
||||
@@ -108,12 +116,62 @@ class TailReader:
|
||||
|
||||
def parse_line(line, export_usernames=False, max_user_labels=100):
|
||||
try:
|
||||
# First handle session closed lines so we can decrement active gauges
|
||||
m_close = pam_session_closed.search(line)
|
||||
if m_close:
|
||||
service = m_close.group('service')
|
||||
username = m_close.group(2)
|
||||
# map service to a source label
|
||||
if service.startswith('sshd') or service == 'sshd':
|
||||
source = 'ssh'
|
||||
elif service.startswith('login') or service == 'login':
|
||||
source = 'tty'
|
||||
elif service.startswith('systemd') or service.startswith('systemd-user'):
|
||||
source = 'systemd-user'
|
||||
elif service.startswith('sudo'):
|
||||
source = 'sudo'
|
||||
else:
|
||||
source = None
|
||||
|
||||
if source:
|
||||
# decrement the aggregated active gauge (but avoid negative values when possible)
|
||||
try:
|
||||
child = active_sessions.labels(source=source)
|
||||
val = child._value.get()
|
||||
if val and val > 0:
|
||||
child.dec()
|
||||
except Exception:
|
||||
# fallback: attempt to decrement anyway
|
||||
try:
|
||||
active_sessions.labels(source=source).dec()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if export_usernames and username in _seen_usernames:
|
||||
try:
|
||||
childu = active_sessions_by_user.labels(source=source, username=username)
|
||||
v = childu._value.get()
|
||||
if v and v > 0:
|
||||
childu.dec()
|
||||
except Exception:
|
||||
try:
|
||||
active_sessions_by_user.labels(source=source, username=username).dec()
|
||||
except Exception:
|
||||
pass
|
||||
# nothing else to do for closed lines
|
||||
return 'closed'
|
||||
|
||||
# ssh (captures method and username)
|
||||
m = pam_ssh_session.search(line)
|
||||
if m:
|
||||
method = m.group(1) # 'password' or 'publickey'
|
||||
username = m.group(2)
|
||||
auth_sessions.labels(source='ssh', method=method).inc()
|
||||
# increment active sessions gauge (aggregated by source)
|
||||
try:
|
||||
active_sessions.labels(source='ssh').inc()
|
||||
except Exception:
|
||||
pass
|
||||
if export_usernames:
|
||||
if username not in _seen_usernames:
|
||||
if len(_seen_usernames) >= max_user_labels:
|
||||
@@ -122,6 +180,10 @@ def parse_line(line, export_usernames=False, max_user_labels=100):
|
||||
_seen_usernames.add(username)
|
||||
if username in _seen_usernames:
|
||||
auth_sessions_by_user.labels(source='ssh', method=method, username=username).inc()
|
||||
try:
|
||||
active_sessions_by_user.labels(source='ssh', username=username).inc()
|
||||
except Exception:
|
||||
pass
|
||||
return 'ssh'
|
||||
|
||||
# tty
|
||||
|
||||
@@ -51,3 +51,45 @@ def test_export_usernames_limited():
|
||||
parse_line(line, export_usernames=True, max_user_labels=3)
|
||||
|
||||
assert len(_seen_usernames) == 3
|
||||
|
||||
|
||||
def test_active_session_inc_dec():
|
||||
"""Ensure active_sessions gauge increments on open and decrements on close."""
|
||||
import sys
|
||||
sys.path.insert(0, str(pathlib.Path(__file__).parent))
|
||||
from exporter import active_sessions, active_sessions_by_user
|
||||
from exporter import _seen_usernames
|
||||
|
||||
# reset state
|
||||
_seen_usernames.clear()
|
||||
# reset gauges by re-creating labels (can't reset Gauge easily; we read internal value)
|
||||
# start from zero
|
||||
s_label = active_sessions.labels(source='ssh')
|
||||
# ensure starting at 0 by reading internal value
|
||||
try:
|
||||
assert s_label._value.get() == 0
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# simulate open
|
||||
open_line = "Oct 2 10:00:00 host sshd[123]: Accepted password for alice from 1.2.3.4 port 123 ssh2"
|
||||
parse_line(open_line, export_usernames=True, max_user_labels=10)
|
||||
|
||||
# now the aggregated gauge should be >= 1
|
||||
try:
|
||||
val = s_label._value.get()
|
||||
assert val >= 1
|
||||
except Exception:
|
||||
# if internals are not accessible on this prometheus client version, at least ensure username tracked
|
||||
assert 'alice' in _seen_usernames
|
||||
|
||||
# simulate close (pam_unix(...): session closed for user alice) with service 'sshd'
|
||||
close_line = "Oct 2 10:00:05 host sshd[123]: pam_unix(sshd:session): session closed for user alice"
|
||||
parse_line(close_line, export_usernames=True, max_user_labels=10)
|
||||
|
||||
# aggregated gauge should not be negative; try to assert decreased
|
||||
try:
|
||||
val2 = s_label._value.get()
|
||||
assert val2 >= 0
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
Auth exporter Grafana dashboard
|
||||
=================================
|
||||
|
||||
This directory contains a Grafana dashboard JSON you can import to visualize the auth-exporter metrics.
|
||||
|
||||
Panels included
|
||||
- Active sessions (by source) — `auth_sessions_active`
|
||||
- Auth sessions started (rate) — `sum by (source, method) (rate(auth_sessions_total[5m]))`
|
||||
- Parse errors (rate) — `rate(auth_exporter_parse_errors_total[5m])`
|
||||
- Exporter uptime — `auth_exporter_uptime_seconds`
|
||||
- User label drops — `rate(auth_exporter_user_labels_dropped_total[5m])`
|
||||
- Top users table (optional, only useful if `--export-usernames` is enabled) — `topk(N, sum by (username) (rate(auth_sessions_by_user_total[5m])))`
|
||||
|
||||
How to import
|
||||
1. In Grafana, go to Dashboards → Import.
|
||||
2. Upload the `auth_exporter_dashboard.json` file or paste its JSON.
|
||||
3. Choose your Prometheus datasource.
|
||||
|
||||
Privacy and performance note
|
||||
- The "Top users" table requires the exporter to be run with `--export-usernames`. That option can generate high-cardinality series and should only be enabled for small, controlled environments or with a username whitelist.
|
||||
@@ -0,0 +1,164 @@
|
||||
{
|
||||
"dashboard": {
|
||||
"id": null,
|
||||
"uid": "auth-exporter",
|
||||
"title": "Auth Log Exporter",
|
||||
"tags": [
|
||||
"exporter",
|
||||
"auth",
|
||||
"prometheus"
|
||||
],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 30,
|
||||
"version": 1,
|
||||
"refresh": "10s",
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"type": "datasource",
|
||||
"name": "datasource",
|
||||
"label": "Prometheus",
|
||||
"query": "prometheus",
|
||||
"current": {
|
||||
"text": "Prometheus",
|
||||
"value": "-- Grafana --"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "interval",
|
||||
"name": "interval",
|
||||
"label": "Interval",
|
||||
"query": "5s,10s,30s,1m,5m",
|
||||
"current": {
|
||||
"text": "1m",
|
||||
"value": "1m"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "textbox",
|
||||
"name": "topn",
|
||||
"label": "Top N users",
|
||||
"query": "10",
|
||||
"current": {
|
||||
"text": "10",
|
||||
"value": "10"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Active sessions (by source)",
|
||||
"id": 1,
|
||||
"datasource": "${datasource}",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "auth_sessions_active",
|
||||
"legendFormat": "{{source}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Auth sessions started (rate)",
|
||||
"id": 2,
|
||||
"datasource": "${datasource}",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (source, method) (rate(auth_sessions_total[5m]))",
|
||||
"legendFormat": "{{source}} / {{method}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 0,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Parse errors (rate)",
|
||||
"id": 3,
|
||||
"datasource": "${datasource}",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(auth_exporter_parse_errors_total[5m])",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 8,
|
||||
"w": 12,
|
||||
"h": 6
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Exporter uptime (s)",
|
||||
"id": 4,
|
||||
"datasource": "${datasource}",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "auth_exporter_uptime_seconds",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 8,
|
||||
"w": 6,
|
||||
"h": 4
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "User label drops (rate)",
|
||||
"id": 5,
|
||||
"datasource": "${datasource}",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(auth_exporter_user_labels_dropped_total[5m])",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 18,
|
||||
"y": 8,
|
||||
"w": 6,
|
||||
"h": 4
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "table",
|
||||
"title": "Top users by session starts (last 5m)",
|
||||
"id": 6,
|
||||
"datasource": "${datasource}",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "topk(int(${topn}), sum by (username) (rate(auth_sessions_by_user_total[5m])))",
|
||||
"legendFormat": "{{username}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 14,
|
||||
"w": 24,
|
||||
"h": 8
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"overwrite": false
|
||||
}
|
||||
Reference in New Issue
Block a user