alexpires.me/a13labs.infra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Add auth exporter module: implement Kubernetes resources, exporter logic, and testing

Browse Source
This commit is contained in:
Alexandre Pires
2025-10-03 00:26:47 +02:00
parent 0a0d4c9c15
commit eb60d37c2d
10 changed files with 489 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
terraform/modules/utils/auth-exporter/config.tf
+15
View File
@@ -0,0 +1,15 @@
locals {
app_version = var.tag
workdir = "${var.persistent_folder}/auth-exporter"
commands = [
"python -m venv /tmp/venv",
". /tmp/venv/bin/activate",
"pip install -r /exporter/requirements.txt",
"python /exporter/exporter.py --export-usernames"
]
exporter_script = file("${path.module}/resources/exporter.py")
requirements_txt = file("${path.module}/resources/requirements.txt")
exporter_checksum = md5(local.exporter_script)
requirements_checksum = md5(local.requirements_txt)
}
terraform/modules/utils/auth-exporter/main.tf
+121
View File
@@ -0,0 +1,121 @@
resource "kubernetes_service_account" "auth_exporter" {
metadata {
name = "auth-exporter-sa"
namespace = var.namespace
}
}
resource "kubernetes_config_map" "auth_exporter" {
metadata {
name = "auth-exporter-config"
namespace = var.namespace
}
data = {
"exporter.py" = local.exporter_script
"requirements.txt" = local.requirements_txt
}
}
resource "kubernetes_deployment" "auth_exporter" {
metadata {
name = "auth-exporter"
namespace = var.namespace
labels = {
app = "auth-exporter"
}
}
spec {
replicas = 1
selector {
match_labels = {
app = "auth-exporter"
}
}
template {
metadata {
labels = {
app = "auth-exporter"
}
annotations = {
"checksum/exporter" = local.exporter_checksum
"checksum/requirements" = local.requirements_checksum
}
}
spec {
service_account_name = kubernetes_service_account.auth_exporter.metadata[0].name
automount_service_account_token = false
container {
name = "auth-exporter"
image = var.tag
command = ["sh", "-c", join(" && ", local.commands)]
port {
container_port = 9100
name = "http-metrics"
}
volume_mount {
name = "exporter-volume"
mount_path = "/exporter"
read_only = true
}
volume_mount {
name = "workdir"
mount_path = "/tmp"
}
volume_mount {
name = "host-logs"
mount_path = "/var/log"
read_only = true
}
security_context {
run_as_user = 0
run_as_group = 0
read_only_root_filesystem = true
}
}
volume {
name = "exporter-volume"
config_map {
name = kubernetes_config_map.auth_exporter.metadata[0].name
}
}
volume {
name = "workdir"
host_path {
path = local.workdir
type = "DirectoryOrCreate"
}
}
volume {
name = "host-logs"
host_path {
path = "/var/log"
type = "Directory"
}
}
}
}
}
}
resource "kubernetes_service" "auth_exporter" {
metadata {
name = "auth-exporter"
namespace = var.namespace
labels = {
app = "auth-exporter"
}
}
spec {
selector = {
app = "auth-exporter"
}
port {
port = 9100
target_port = 9100
protocol = "TCP"
name = "http-metrics"
}
type = "ClusterIP"
}
}
terraform/modules/utils/auth-exporter/provider.tf
+9
View File
@@ -0,0 +1,9 @@
terraform {
required_version = "~>1.8"
required_providers {
kubernetes = {
source = "hashicorp/kubernetes"
version = "2.36.0"
}
}
}
terraform/modules/utils/auth-exporter/resources/exporter.py
+267
View File
@@ -0,0 +1,267 @@
#!/usr/bin/env python3
import os
import signal
import argparse
import logging
import re
import time
from threading import Thread
from wsgiref.simple_server import make_server
from prometheus_client import make_wsgi_app, Counter, Gauge
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
# Aggregated, labeled metric (safe: labels have small domain)
auth_sessions = Counter(
'auth_sessions_total',
'Total auth sessions started',
['source', 'method']
)
# Optional, user-labeled metric. Disabled by default because usernames are high-cardinality.
auth_sessions_by_user = Counter(
'auth_sessions_by_user_total',
'Total auth sessions by username (high-cardinality — enable with --export-usernames)',
['source', 'method', 'username']
)
# exporter health/observability
parse_errors = Counter('auth_exporter_parse_errors_total', 'Total parse errors')
uptime = Gauge('auth_exporter_uptime_seconds', 'Exporter uptime in seconds')
user_label_dropped = Counter('auth_exporter_user_labels_dropped_total', 'Number of username labels dropped due to max-user-labels limit')
# more tolerant username pattern (allows -, ., @, digits)
USERNAME = r'[\w\-\.\@]+'
IP = r'[\d\.]+|[\da-fA-F:]+'
pam_ssh_session = re.compile(
rf'sshd\[\d+\]: Accepted (password|publickey) for ({USERNAME}) from ({IP}) port \d+ ssh2'
)
tty_pattern = re.compile(
rf'login\[\d+\]: pam_unix\(login:session\): session opened for user ({USERNAME}) by \(uid=\d+\)'
)
systemd_logind_session = re.compile(
rf'systemd-logind\[\d+\]: New session \d+ of user ({USERNAME})\.'
)
pam_systemd_session = re.compile(
rf'systemd: pam_unix\(systemd-user:session\): session opened for user ({USERNAME})(?:\(uid=\d+\))? by \(uid=\d+\)'
)
pam_sudo_session = re.compile(
rf'sudo: pam_unix\(sudo[^)]*:session\): session opened for user ({USERNAME})(?:\(uid=\d+\))? by (?:({USERNAME})|uid=\d+|\(uid=\d+\))'
)
# Track seen usernames to limit cardinality if username export is enabled
_seen_usernames = set()
class TailReader:
def __init__(self, path, start_at_end=True):
self.path = path
self.file = None
self.inode = None
self.start_at_end = start_at_end
self.open_file()
def open_file(self):
try:
self.file = open(self.path, 'r', errors='ignore')
stat = os.fstat(self.file.fileno())
self.inode = (stat.st_ino, stat.st_dev)
if self.start_at_end:
self.file.seek(0, os.SEEK_END)
logging.info("Opened %s (inode=%s)", self.path, self.inode)
except Exception as e:
logging.error("Failed to open %s: %s", self.path, e)
raise
def check_rotate(self):
try:
stat = os.stat(self.path)
current = (stat.st_ino, stat.st_dev)
if current != self.inode:
logging.info("Detected rotation/truncate for %s (old=%s new=%s)", self.path, self.inode, current)
self.file.close()
self.open_file()
else:
# handle copytruncate: file same inode but possibly truncated
try:
pos = self.file.tell()
if pos > stat.st_size:
logging.info("Detected file truncated for %s (pos=%d size=%d), seeking to %d", self.path, pos, stat.st_size, stat.st_size)
self.file.seek(stat.st_size, os.SEEK_SET)
except Exception as e:
logging.error("Error checking for truncation: %s", e)
except FileNotFoundError:
# file temporarily missing (rotation); try reopen later
logging.warning("File %s not found while monitoring; will retry", self.path)
time.sleep(1)
except Exception as e:
logging.error("Error checking rotation: %s", e)
def follow_once(self):
# yield any new lines
while True:
line = self.file.readline()
if not line:
break
yield line
def parse_line(line, export_usernames=False, max_user_labels=100):
try:
# ssh (captures method and username)
m = pam_ssh_session.search(line)
if m:
method = m.group(1) # 'password' or 'publickey'
username = m.group(2)
auth_sessions.labels(source='ssh', method=method).inc()
if export_usernames:
if username not in _seen_usernames:
if len(_seen_usernames) >= max_user_labels:
user_label_dropped.inc()
else:
_seen_usernames.add(username)
if username in _seen_usernames:
auth_sessions_by_user.labels(source='ssh', method=method, username=username).inc()
return 'ssh'
# tty
m = tty_pattern.search(line)
if m:
username = m.group(1)
auth_sessions.labels(source='tty', method='local').inc()
if export_usernames:
if username not in _seen_usernames:
if len(_seen_usernames) >= max_user_labels:
user_label_dropped.inc()
else:
_seen_usernames.add(username)
if username in _seen_usernames:
auth_sessions_by_user.labels(source='tty', method='local', username=username).inc()
return 'tty'
# systemd-logind
m = systemd_logind_session.search(line)
if m:
username = m.group(1)
auth_sessions.labels(source='systemd-logind', method='pam').inc()
if export_usernames:
if username not in _seen_usernames:
if len(_seen_usernames) >= max_user_labels:
user_label_dropped.inc()
else:
_seen_usernames.add(username)
if username in _seen_usernames:
auth_sessions_by_user.labels(source='systemd-logind', method='pam', username=username).inc()
return 'systemd-logind'
# systemd user session via pam
m = pam_systemd_session.search(line)
if m:
username = m.group(1)
auth_sessions.labels(source='systemd-user', method='pam').inc()
if export_usernames:
if username not in _seen_usernames:
if len(_seen_usernames) >= max_user_labels:
user_label_dropped.inc()
else:
_seen_usernames.add(username)
if username in _seen_usernames:
auth_sessions_by_user.labels(source='systemd-user', method='pam', username=username).inc()
return 'systemd-user'
# sudo
m = pam_sudo_session.search(line)
if m:
# sudo regex captures the target username (group 1) or similar; normalize to 'sudo'
username = m.group(1)
auth_sessions.labels(source='sudo', method='sudo').inc()
if export_usernames:
if username not in _seen_usernames:
if len(_seen_usernames) >= max_user_labels:
user_label_dropped.inc()
else:
_seen_usernames.add(username)
if username in _seen_usernames:
auth_sessions_by_user.labels(source='sudo', method='sudo', username=username).inc()
return 'sudo'
except Exception:
parse_errors.inc()
logging.exception("Error parsing line")
return None
running = True
start_time = time.time()
httpd = None
_server_thread = None
def handle_signal(signum, frame):
global running, httpd
logging.info("Received signal %s, shutting down", signum)
running = False
try:
if httpd:
logging.info("Shutting down HTTP server")
httpd.shutdown()
except Exception:
logging.exception("Error shutting down HTTP server")
def main():
global running
parser = argparse.ArgumentParser(description="Auth log prometheus exporter")
parser.add_argument('--logfile', default='/var/log/auth.log', help='Path to auth log')
parser.add_argument('--port', type=int, default=9100, help='Prometheus metrics port')
parser.add_argument('--start-at-beginning', action='store_true', help='Parse file from beginning on first run')
parser.add_argument('--interval', type=float, default=1.0, help='Poll interval seconds')
parser.add_argument('--export-usernames', action='store_true', help='Enable exporting username labels (DANGEROUS: may increase cardinality)')
parser.add_argument('--max-user-labels', type=int, default=100, help='Maximum distinct username labels to export when --export-usernames is enabled')
args = parser.parse_args()
# NOTE: reading /var/log/auth.log usually requires root
if not os.access(args.logfile, os.R_OK):
logging.warning("No read access to %s. Exporter may need to run as root or use systemd-journal.", args.logfile)
# Start Prometheus WSGI app in a daemon thread so it does not block process exit.
global httpd, _server_thread
app = make_wsgi_app()
httpd = make_server('', args.port, app)
_server_thread = Thread(target=httpd.serve_forever, name="prometheus-http")
_server_thread.daemon = True
_server_thread.start()
logging.info("Starting Prometheus session exporter on port %d...", args.port)
tail = TailReader(args.logfile, start_at_end=not args.start_at_beginning)
signal.signal(signal.SIGTERM, handle_signal)
signal.signal(signal.SIGINT, handle_signal)
try:
while running:
tail.check_rotate()
for line in tail.follow_once():
parse_line(line, export_usernames=args.export_usernames, max_user_labels=args.max_user_labels)
uptime.set(time.time() - start_time)
time.sleep(args.interval)
except Exception:
logging.exception("Unhandled error in main loop")
finally:
logging.info("Shutting down exporter, cleaning up resources")
try:
if tail.file:
tail.file.close()
except Exception:
logging.exception("Error closing tail file")
try:
if httpd:
httpd.shutdown()
except Exception:
logging.exception("Error shutting down HTTP server")
try:
if _server_thread:
_server_thread.join(timeout=2)
except Exception:
logging.exception("Error joining HTTP server thread")
logging.info("Exporter stopped")
if __name__ == '__main__':
main()
terraform/modules/utils/auth-exporter/resources/exporter_test.py
+53
View File
@@ -0,0 +1,53 @@
import pathlib
from exporter import parse_line
def test_parse_samples():
"""Ensure representative log lines are parsed to the expected source."""
samples = [
("Oct 2 10:00:00 host sshd[123]: Accepted password for alice from 192.0.2.1 port 12345 ssh2", 'ssh'),
("Oct 2 10:00:00 host sshd[124]: Accepted publickey for bob from 2001:db8::1 port 54321 ssh2", 'ssh'),
("Oct 2 10:00:00 host login[200]: pam_unix(login:session): session opened for user root by (uid=0)", 'tty'),
("Oct 2 10:00:00 host systemd-logind[735]: New session 42 of user alice.", 'systemd-logind'),
("Oct 2 10:00:00 host systemd: pam_unix(systemd-user:session): session opened for user alice(uid=1000) by (uid=0)", 'systemd-user'),
("Oct 2 10:00:00 host sudo: pam_unix(sudo:session): session opened for user root by (uid=1000)", 'sudo'),
]
for line, expected in samples:
assert parse_line(line, export_usernames=False) == expected
def test_fixture_parsing_no_cron_misclassified():
"""Parse the provided fixtures/auth.log and ensure cron session lines are not mis-classified as tty/systemd/sudo/ssh."""
fixtures = pathlib.Path(__file__).parent / 'fixtures' / 'auth.log'
assert fixtures.exists(), f"fixture file missing: {fixtures}"
with fixtures.open('r') as fh:
lines = fh.readlines()
# Find a cron session line in the fixture and ensure it does not parse as a known auth session
cron_line = None
for l in lines:
if 'pam_unix(cron:session): session opened for user' in l:
cron_line = l
break
assert cron_line is not None, "cron sample line missing in fixture"
assert parse_line(cron_line, export_usernames=False) is None
def test_export_usernames_limited():
"""When exporting usernames, the exporter should limit unique username labels to max_user_labels."""
# import private set to inspect size; tests run in same process so this is fine
from exporter import _seen_usernames
# clear any previous state
_seen_usernames.clear()
# feed 5 distinct usernames but allow only 3 labels
for i in range(5):
line = f"Oct 2 10:00:00 host sshd[123]: Accepted password for user{i} from 1.2.3.{i} port 123 ssh2"
parse_line(line, export_usernames=True, max_user_labels=3)
assert len(_seen_usernames) == 3
terraform/modules/utils/auth-exporter/resources/requirements.txt
+1
View File
@@ -0,0 +1 @@
prometheus_client==0.23.1
terraform/modules/utils/auth-exporter/variables.tf
+16
View File
@@ -0,0 +1,16 @@
variable "tag" {
description = "Python image tag"
type = string
default = "python:3.12-slim"
}
variable "namespace" {
description = "Kubernetes namespace to deploy the prometheus exporter"
type = string
default = "kube-system"
}
variable "persistent_folder" {
description = "The path to the persistent folder"
type = string
}
terraform/modules/utils/cert_checker/variables.tf
+1 -1
View File
@@ -5,7 +5,7 @@ variable "tag" {
}
variable "namespace" {
description = "Kubernetes namespace to deploy the metrics server"
description = "Kubernetes namespace to deploy the cert checker"
type = string
}