Add Prometheus and Alertmanager configurations: implement custom rules, alerting, and Telegram notifications

terraform/apps/prod-01/apps.tf

@@ -132,8 +132,18 @@ module "proxy" {
 module "prometheus" {
   source = "../../modules/utils/prometheus"
 
-  persistent_folder = "${local.persistent_folder}/monitoring"
-  config            = file("${path.module}/resources/prometheus.yml")
+  cluster_name        = "prod-01"
+  persistent_folder   = "${local.persistent_folder}/monitoring"
+  config              = file("${path.module}/resources/prometheus.yml")
+  rules               = local.prometheus_rules
+  alertmanager_config = local.alertmanager_config
+
+  email_config = local.alertmanager_email_config
+  telegram_config = {
+    bot_token = var.telegram_bot_token
+    chat_id   = local.telegram_chat_id
+    message   = local.telegram_message
+  }
 }
 
 module "fail2ban_exporter" {

terraform/apps/prod-01/config.tf

@@ -192,6 +192,21 @@ locals {
     }
   }
 
+  # prometheus
+  prometheus_config   = file("${path.module}/resources/prometheus.yml")
+  prometheus_rules    = file("${path.module}/resources/prometheus-rules.yml")
+  alertmanager_config = file("${path.module}/resources/alertmanager.yml")
+  alertmanager_email_config = {
+    to            = "c.alexandre.pires@alexpires.me"
+    from          = "alertmanager@${local.primary_domain}"
+    smarthost     = "smtp.tem.scw.cloud:587"
+    auth_username = var.scaleway_project_id
+    auth_password = local.mail_app_api_secret_key
+  }
+
+  telegram_chat_id = "5432313610" # a13labs channel
+  telegram_message = file("${path.module}/resources/telegram.tpl")
+
   # get secrets from remote state
   backup_bucket_name        = data.terraform_remote_state.scaleway.outputs.backup_bucket_name
   backup_app_api_secret_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_secret_key

terraform/apps/prod-01/resources/alertmanager.yml

@@ -0,0 +1,11 @@
+route:
+  group_by: ["alertname"]
+  receiver: "default"
+  group_wait: 10s
+  group_interval: 10m
+  repeat_interval: 1h
+  routes:
+    - receiver: "telegram"
+      matchers:
+        - 'alertname="UserLoggedIn"'
+      continue: false

terraform/apps/prod-01/resources/prometheus-rules.yml

@@ -0,0 +1,10 @@
+groups:
+  - name: auth.rules
+    rules:
+      - alert: UserLoggedIn
+        expr: increase(auth_sessions_active_by_user[1m]) > 0
+        labels:
+          severity: info
+        annotations:
+          summary: "Login: {{ $labels.username }} on {{ $labels.instance }}"
+          description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}."

terraform/apps/prod-01/resources/prometheus.yml

@@ -1,41 +1,4 @@
-global:
-  scrape_interval: 5s
-  evaluation_interval: 5s
-rule_files:
-  - /etc/prometheus/prometheus.rules
 scrape_configs:
-  - job_name: kubernetes-nodes-cadvisor
-    scrape_interval: 10s
-    scrape_timeout: 10s
-    scheme: https # remove if you want to scrape metrics on insecure port
-    tls_config:
-      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-      insecure_skip_verify: true
-    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-    kubernetes_sd_configs:
-      - role: node
-    relabel_configs:
-      - action: labelmap
-        regex: __meta_kubernetes_node_label_(.+)
-      # Only for Kubernetes ^1.7.3.
-      # See: https://github.com/prometheus/prometheus/issues/2916
-      - target_label: __address__
-        replacement: kubernetes.default.svc:443
-      - source_labels: [__meta_kubernetes_node_name]
-        regex: (.+)
-        target_label: __metrics_path__
-        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
-    metric_relabel_configs:
-      - action: replace
-        source_labels: [id]
-        regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$'
-        target_label: rkt_container_name
-        replacement: "${2}-${1}"
-      - action: replace
-        source_labels: [id]
-        regex: '^/system\.slice/(.+)\.service$'
-        target_label: systemd_service_name
-        replacement: "${1}"
   - job_name: node-exporter
     scrape_interval: 10s
     static_configs:

terraform/apps/prod-01/resources/telegram.tpl

@@ -0,0 +1,13 @@
+{{/* Always produce a non-empty message. Handles single or multiple firing alerts and missing username. */}}
+{{ if gt (len .Alerts.Firing) 1 -}}
+<b>{{ len .Alerts.Firing }} logins</b> on {{ with .CommonLabels.cluster }}<b>{{ . }}</b>{{ else }}<b>unknown host</b>{{ end }}:
+{{- range .Alerts.Firing }}
+• {{ with .Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} — Source: <code>{{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with .Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
+{{- end }}
+{{ else if eq (len .Alerts.Firing) 1 -}}
+{{ $a := index .Alerts.Firing 0 -}}
+<b>Login:</b> {{ with $a.Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} on <b>{{ $a.Labels.cluster }}</b>
+Source: <code>{{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with $a.Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
+{{ else -}}
+<b>Login detected</b> — details unavailable.
+{{ end -}}

terraform/apps/prod-01/secrets.tf

@@ -128,3 +128,9 @@ variable "nginx_token_bearer" {
   type        = string
   sensitive   = true
 }
+
+variable "telegram_bot_token" {
+  description = "Telegram bot token for Alertmanager notifications"
+  type        = string
+  sensitive   = true
+}

terraform/apps/prod-01/sectool.env

@@ -32,4 +32,4 @@ TF_VAR_wireguard_1_private_key=$WIREGUARD_1_PRIVATE_KEY
 TF_VAR_wireguard_1_public_key=$WIREGUARD_1_PUBLIC_KEY
 TF_VAR_wireguard_1_pre_shared_key=$WIREGUARD_1_PRE_SHARED_KEY
 TF_VAR_nginx_token_bearer=$NGINX_TOKEN_BEARER
-
+TF_VAR_telegram_bot_token=$TELEGRAM_BOT_TOKEN