alexpires.me/a13labs.infra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: add disk exporter module and related configurations

Browse Source 
- Introduced a new disk exporter module to monitor disk health using SMART data.
- Updated `apps.tf` to include the disk exporter module with necessary parameters.
- Added local variables for smartctl version and checksum in `config.tf`.
- Enhanced `alertmanager.yml` to handle DiskUnhealthy alerts with specific grouping and dispatch settings.
- Created new Prometheus rules for monitoring disk health, including alerts for unhealthy disks.
- Updated Prometheus configuration to scrape metrics from the new disk exporter.
- Enhanced Telegram notification template to include resolved alerts and improved formatting.
- Updated the auth-exporter module to specify the Python image tag correctly.
- Created the disk-exporter module with Kubernetes resources, including deployment, service account, and config map.
- Added necessary Python scripts and requirements for the disk exporter functionality.
This commit is contained in:
Alexandre Pires
2025-10-06 18:41:01 +02:00
parent 0e167d747d
commit 429a94332e
19 changed files with 766 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
terraform/apps/dev-01/apps.tf
+9
View File
@@ -81,3 +81,12 @@ module "auth_exporter" {
source = "../../modules/utils/auth-exporter"
persistent_folder = "${local.persistent_folder}/monitoring"
}
module "disk_exporter" {
source = "../../modules/utils/disk-exporter"
persistent_folder = "${local.persistent_folder}/monitoring"
smartctl_static_version = local.smartctl_static_version
smartctl_sha256 = local.smartctl_sha256
check_frequency = 600
standby_mode = false
}
terraform/apps/dev-01/config.tf
+3
View File
@@ -184,6 +184,9 @@ locals {
auth_password = local.mail_app_api_secret_key
}
smartctl_static_version = "8.0-198-g34e7e4975211"
smartctl_sha256 = "d7e779664248942bb519b254dcb49cddaec2593e192cc468254c58e64da20840"
telegram_chat_id = "5432313610" # a13labs channel
telegram_message = file("${path.module}/resources/telegram.tpl")
terraform/apps/dev-01/resources/alertmanager.yml
+12
View File
@@ -8,4 +8,16 @@ route:
- receiver: "telegram"
matchers:
- 'alertname="UserLoggedIn"'
group_by: ["alertname", "username"] # add username to avoid batching different users
group_wait: 0s # immediate dispatch
group_interval: 2m # short regroup window
repeat_interval: 30m # optional: adjust reminder cadence
continue: false
- receiver: "telegram"
matchers:
- 'alertname="DiskUnhealthy"'
group_by: ["alertname", "device"] # add device to avoid batching different disks
group_wait: 0s # immediate dispatch
group_interval: 2m # short regroup window
repeat_interval: 30m # optional: adjust reminder cadence
continue: false
terraform/apps/dev-01/resources/prometheus-rules.yml
+13 -3
View File
@@ -2,9 +2,19 @@ groups:
- name: auth.rules
rules:
- alert: UserLoggedIn
expr: increase(auth_sessions_active_by_user[1m]) > 0
expr: changes(auth_sessions_active_by_user[1m]) > 0
labels:
severity: info
annotations:
summary: "Login: {{ $labels.username }} on {{ $labels.instance }}"
description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}."
summary: "Login: {{ $labels.username }} on dev-01"
description: "User {{ $labels.username }} logged in via {{ $labels.source }} on dev-01."
- name: disk.rules
rules:
- alert: DiskUnhealthy
expr: count(disk_info{})-count(disk_healthy{}) > 0
for: 2m
labels:
severity: critical
annotations:
summary: "Disk unhealthy: {{ $labels.device }} on dev-01"
description: "SMART overall health failed for device {{ $labels.device }} (disk_healthy=0). Investigate immediately."
terraform/apps/dev-01/resources/prometheus.yml
+11
View File
@@ -5,8 +5,19 @@ scrape_configs:
- targets:
- "node-exporter.prometheus.svc.cluster.local:9100"
- "10.19.4.1:9100"
- labels:
cluster: "dev-01"
- job_name: auth-exporter
scrape_interval: 10s
static_configs:
- targets:
- "auth-exporter.prometheus.svc.cluster.local:9100"
- labels:
cluster: "dev-01"
- job_name: disk-exporter
scrape_interval: 120s
static_configs:
- targets:
- "disk-exporter.prometheus.svc.cluster.local:9100"
- labels:
cluster: "dev-01"
terraform/apps/dev-01/resources/telegram.tpl
+29 -9
View File
@@ -1,13 +1,33 @@
{{/* Always produce a non-empty message. Handles single or multiple firing alerts and missing username. */}}
{{ if gt (len .Alerts.Firing) 1 -}}
<b>{{ len .Alerts.Firing }} logins</b> on {{ with .CommonLabels.cluster }}<b>{{ . }}</b>{{ else }}<b>unknown host</b>{{ end }}:
{{/* Template enhanced to include resolved alerts while using annotations. */}}
{{ $firingCount := len .Alerts.Firing -}}
{{ $resolvedCount := len .Alerts.Resolved -}}
{{ if or (gt $firingCount 0) (gt $resolvedCount 0) -}}
{{ if and (gt $firingCount 0) (gt $resolvedCount 0) -}}
<b>{{ $firingCount }} firing</b> / <b>{{ $resolvedCount }} resolved</b>
{{ if gt $firingCount 0 -}}
<b>Firing:</b>
{{- range .Alerts.Firing }}
• {{ with .Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} — Source: <code>{{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with .Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
{{- end }}
{{ else if eq (len .Alerts.Firing) 1 -}}
{{ $a := index .Alerts.Firing 0 -}}
<b>Login:</b> {{ with $a.Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} on <b>{{ $a.Labels.cluster }}</b>
Source: <code>{{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with $a.Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
{{ end -}}
{{ if gt $resolvedCount 0 -}}
<b>Resolved:</b>
{{- range .Alerts.Resolved }}
• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
{{- end }}
{{ end -}}
{{ else if gt $firingCount 0 -}}
<b>{{ $firingCount }} alert{{ if ne $firingCount 1 }}s{{ end }} firing</b>:
{{- range .Alerts.Firing }}
• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
{{- end }}
{{ else -}}
<b>{{ $resolvedCount }} alert{{ if ne $resolvedCount 1 }}s{{ end }} resolved</b>:
{{- range .Alerts.Resolved }}
• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
{{- end }}
{{ end -}}
{{ else -}}
<b>Login detected</b> — details unavailable.
<b>No alert data</b> — nothing firing or recently resolved.
{{ end -}}
terraform/apps/prod-01/resources/alertmanager.yml
+4
View File
@@ -8,4 +8,8 @@ route:
- receiver: "telegram"
matchers:
- 'alertname="UserLoggedIn"'
group_by: ["alertname", "username"] # add username to avoid batching different users
group_wait: 0s # immediate dispatch
group_interval: 2m # short regroup window
repeat_interval: 30m # optional: adjust reminder cadence
continue: false
terraform/apps/prod-01/resources/prometheus-rules.yml
+2 -2
View File
@@ -6,5 +6,5 @@ groups:
labels:
severity: info
annotations:
summary: "Login: {{ $labels.username }} on {{ $labels.instance }}"
description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}."
summary: "Login: {{ $labels.username }} on prod-01"
description: "User {{ $labels.username }} logged in via {{ $labels.source }} on prod-01."
terraform/apps/prod-01/resources/prometheus.yml
+6
View File
@@ -4,13 +4,19 @@ scrape_configs:
static_configs:
- targets:
- "node-exporter.prometheus.svc.cluster.local:9100"
labels:
cluster: "prod-01"
- job_name: fail2ban
scrape_interval: 60s
static_configs:
- targets:
- "fail2ban-exporter.prometheus.svc.cluster.local:9100"
labels:
cluster: "prod-01"
- job_name: auth-exporter
scrape_interval: 10s
static_configs:
- targets:
- "auth-exporter.prometheus.svc.cluster.local:9100"
labels:
cluster: "prod-01"
terraform/apps/prod-01/resources/telegram.tpl
+28 -23
View File
@@ -1,28 +1,33 @@
{{- /* Resolved alerts (if any) */ -}}
{{- if gt (len .Alerts.Resolved) 1 -}}
<b>{{ len .Alerts.Resolved }} logins resolved</b> on {{ with .CommonLabels.cluster }}<b>{{ . }}</b>{{ else }}<b>unknown host</b>{{ end }}:
{{- range .Alerts.Resolved }}
• {{ with .Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} — Source: <code>{{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with .Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
{{- end }}
{{- else if eq (len .Alerts.Resolved) 1 -}}
{{ $a := index .Alerts.Resolved 0 -}}
<b>Resolved login:</b> {{ with $a.Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} on <b>{{ $a.Labels.cluster }}</b>
Source: <code>{{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with $a.Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
{{- end -}}
{{/* Template enhanced to include resolved alerts while using annotations. */}}
{{ $firingCount := len .Alerts.Firing -}}
{{ $resolvedCount := len .Alerts.Resolved -}}
{{- /* Firing alerts (original logic) */ -}}
{{ if gt (len .Alerts.Firing) 1 -}}
<b>{{ len .Alerts.Firing }} logins</b> on {{ with .CommonLabels.cluster }}<b>{{ . }}</b>{{ else }}<b>unknown host</b>{{ end }}:
{{ if or (gt $firingCount 0) (gt $resolvedCount 0) -}}
{{ if and (gt $firingCount 0) (gt $resolvedCount 0) -}}
<b>{{ $firingCount }} firing</b> / <b>{{ $resolvedCount }} resolved</b>
{{ if gt $firingCount 0 -}}
<b>Firing:</b>
{{- range .Alerts.Firing }}
• {{ with .Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} — Source: <code>{{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with .Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
{{- end }}
{{ else if eq (len .Alerts.Firing) 1 -}}
{{ $a := index .Alerts.Firing 0 -}}
<b>Login:</b> {{ with $a.Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} on <b>{{ $a.Labels.cluster }}</b>
Source: <code>{{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with $a.Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
{{ else -}}
{{- /* If there were resolved alerts above this will still show when no firing alerts exist. */ -}}
{{ if eq (len .Alerts.Resolved) 0 -}}
<b>Login detected</b> — details unavailable.
{{ end -}}
{{ if gt $resolvedCount 0 -}}
<b>Resolved:</b>
{{- range .Alerts.Resolved }}
• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
{{- end }}
{{ end -}}
{{ else if gt $firingCount 0 -}}
<b>{{ $firingCount }} alert{{ if ne $firingCount 1 }}s{{ end }} firing</b>:
{{- range .Alerts.Firing }}
• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
{{- end }}
{{ else -}}
<b>{{ $resolvedCount }} alert{{ if ne $resolvedCount 1 }}s{{ end }} resolved</b>:
{{- range .Alerts.Resolved }}
• <b>{{ or .Annotations.summary "No summary" }}</b>{{ with .Annotations.description }} — {{ . }}{{ end }}
{{- end }}
{{ end -}}
{{ else -}}
<b>No alert data</b> — nothing firing or recently resolved.
{{ end -}}