Add Prometheus and Alertmanager configurations: implement custom rules, alerting, and Telegram notifications

terraform/apps/prod-01/resources/alertmanager.yml

@@ -0,0 +1,11 @@
+route:
+  group_by: ["alertname"]
+  receiver: "default"
+  group_wait: 10s
+  group_interval: 10m
+  repeat_interval: 1h
+  routes:
+    - receiver: "telegram"
+      matchers:
+        - 'alertname="UserLoggedIn"'
+      continue: false

terraform/apps/prod-01/resources/prometheus-rules.yml

@@ -0,0 +1,10 @@
+groups:
+  - name: auth.rules
+    rules:
+      - alert: UserLoggedIn
+        expr: increase(auth_sessions_active_by_user[1m]) > 0
+        labels:
+          severity: info
+        annotations:
+          summary: "Login: {{ $labels.username }} on {{ $labels.instance }}"
+          description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}."

terraform/apps/prod-01/resources/prometheus.yml

@@ -1,41 +1,4 @@
-global:
-  scrape_interval: 5s
-  evaluation_interval: 5s
-rule_files:
-  - /etc/prometheus/prometheus.rules
 scrape_configs:
-  - job_name: kubernetes-nodes-cadvisor
-    scrape_interval: 10s
-    scrape_timeout: 10s
-    scheme: https # remove if you want to scrape metrics on insecure port
-    tls_config:
-      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-      insecure_skip_verify: true
-    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-    kubernetes_sd_configs:
-      - role: node
-    relabel_configs:
-      - action: labelmap
-        regex: __meta_kubernetes_node_label_(.+)
-      # Only for Kubernetes ^1.7.3.
-      # See: https://github.com/prometheus/prometheus/issues/2916
-      - target_label: __address__
-        replacement: kubernetes.default.svc:443
-      - source_labels: [__meta_kubernetes_node_name]
-        regex: (.+)
-        target_label: __metrics_path__
-        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
-    metric_relabel_configs:
-      - action: replace
-        source_labels: [id]
-        regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$'
-        target_label: rkt_container_name
-        replacement: "${2}-${1}"
-      - action: replace
-        source_labels: [id]
-        regex: '^/system\.slice/(.+)\.service$'
-        target_label: systemd_service_name
-        replacement: "${1}"
   - job_name: node-exporter
     scrape_interval: 10s
     static_configs:

terraform/apps/prod-01/resources/telegram.tpl

@@ -0,0 +1,13 @@
+{{/* Always produce a non-empty message. Handles single or multiple firing alerts and missing username. */}}
+{{ if gt (len .Alerts.Firing) 1 -}}
+<b>{{ len .Alerts.Firing }} logins</b> on {{ with .CommonLabels.cluster }}<b>{{ . }}</b>{{ else }}<b>unknown host</b>{{ end }}:
+{{- range .Alerts.Firing }}
+• {{ with .Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} — Source: <code>{{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with .Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
+{{- end }}
+{{ else if eq (len .Alerts.Firing) 1 -}}
+{{ $a := index .Alerts.Firing 0 -}}
+<b>Login:</b> {{ with $a.Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} on <b>{{ $a.Labels.cluster }}</b>
+Source: <code>{{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with $a.Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
+{{ else -}}
+<b>Login detected</b> — details unavailable.
+{{ end -}}