Add Prometheus and Alertmanager configurations: implement custom rules, alerting, and Telegram notifications
This commit is contained in:
@@ -52,8 +52,18 @@ module "comfyui_proxy" {
|
||||
module "prometheus" {
|
||||
source = "../../modules/utils/prometheus"
|
||||
|
||||
cluster_name = "dev-01"
|
||||
persistent_folder = "${local.persistent_folder}/monitoring"
|
||||
config = file("${path.module}/resources/prometheus.yml")
|
||||
config = local.prometheus_config
|
||||
rules = local.prometheus_rules
|
||||
alertmanager_config = local.alertmanager_config
|
||||
|
||||
email_config = local.alertmanager_email_config
|
||||
telegram_config = {
|
||||
bot_token = var.telegram_bot_token
|
||||
chat_id = local.telegram_chat_id
|
||||
message = local.telegram_message
|
||||
}
|
||||
}
|
||||
|
||||
module "grafana" {
|
||||
|
||||
@@ -167,16 +167,22 @@ locals {
|
||||
backup_app_api_access_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_access_key
|
||||
mail_app_api_secret_key = data.terraform_remote_state.scaleway.outputs.mail_app_api_secret_key
|
||||
|
||||
grafana_fqdn = "grafana.${local.lab_domain}"
|
||||
# prometheus and grafana
|
||||
prometheus_config = file("${path.module}/resources/prometheus.yml")
|
||||
prometheus_rules = file("${path.module}/resources/prometheus-rules.yml")
|
||||
alertmanager_config = file("${path.module}/resources/alertmanager.yml")
|
||||
alertmanager_email_config = {
|
||||
to = "c.alexandre.pires@alexpires.me"
|
||||
from = "alertmanager@${local.lab_domain}"
|
||||
smarthost = "smtp.tem.scw.cloud:587"
|
||||
auth_username = var.scaleway_project_id
|
||||
auth_password = local.mail_app_api_secret_key
|
||||
}
|
||||
|
||||
remote_targets = {
|
||||
"prometheus" = {
|
||||
listen_port = 9090
|
||||
target_host = "10.5.5.5"
|
||||
target_port = 9090
|
||||
protocol = "TCP"
|
||||
}
|
||||
}
|
||||
telegram_chat_id = "5432313610" # a13labs channel
|
||||
telegram_message = file("${path.module}/resources/telegram.tpl")
|
||||
|
||||
grafana_fqdn = "grafana.${local.lab_domain}"
|
||||
}
|
||||
|
||||
data "terraform_remote_state" "scaleway" {
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
route:
|
||||
group_by: ["alertname"]
|
||||
receiver: "default"
|
||||
group_wait: 10s
|
||||
group_interval: 10m
|
||||
repeat_interval: 1h
|
||||
routes:
|
||||
- receiver: "telegram"
|
||||
matchers:
|
||||
- 'alertname="UserLoggedIn"'
|
||||
continue: false
|
||||
@@ -0,0 +1,10 @@
|
||||
groups:
|
||||
- name: auth.rules
|
||||
rules:
|
||||
- alert: UserLoggedIn
|
||||
expr: increase(auth_sessions_active_by_user[1m]) > 0
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: "Login: {{ $labels.username }} on {{ $labels.instance }}"
|
||||
description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}."
|
||||
@@ -1,41 +1,4 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
rule_files:
|
||||
- /etc/prometheus/prometheus.rules
|
||||
scrape_configs:
|
||||
- job_name: kubernetes-nodes-cadvisor
|
||||
scrape_interval: 10s
|
||||
scrape_timeout: 10s
|
||||
scheme: https # remove if you want to scrape metrics on insecure port
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
insecure_skip_verify: true
|
||||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
relabel_configs:
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_node_label_(.+)
|
||||
# Only for Kubernetes ^1.7.3.
|
||||
# See: https://github.com/prometheus/prometheus/issues/2916
|
||||
- target_label: __address__
|
||||
replacement: kubernetes.default.svc:443
|
||||
- source_labels: [__meta_kubernetes_node_name]
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
|
||||
metric_relabel_configs:
|
||||
- action: replace
|
||||
source_labels: [id]
|
||||
regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$'
|
||||
target_label: rkt_container_name
|
||||
replacement: "${2}-${1}"
|
||||
- action: replace
|
||||
source_labels: [id]
|
||||
regex: '^/system\.slice/(.+)\.service$'
|
||||
target_label: systemd_service_name
|
||||
replacement: "${1}"
|
||||
- job_name: node-exporter
|
||||
scrape_interval: 10s
|
||||
static_configs:
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
{{/* Always produce a non-empty message. Handles single or multiple firing alerts and missing username. */}}
|
||||
{{ if gt (len .Alerts.Firing) 1 -}}
|
||||
<b>{{ len .Alerts.Firing }} logins</b> on {{ with .CommonLabels.cluster }}<b>{{ . }}</b>{{ else }}<b>unknown host</b>{{ end }}:
|
||||
{{- range .Alerts.Firing }}
|
||||
• {{ with .Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} — Source: <code>{{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with .Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
|
||||
{{- end }}
|
||||
{{ else if eq (len .Alerts.Firing) 1 -}}
|
||||
{{ $a := index .Alerts.Firing 0 -}}
|
||||
<b>Login:</b> {{ with $a.Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} on <b>{{ $a.Labels.cluster }}</b>
|
||||
Source: <code>{{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with $a.Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
|
||||
{{ else -}}
|
||||
<b>Login detected</b> — details unavailable.
|
||||
{{ end -}}
|
||||
@@ -15,3 +15,9 @@ variable "nextcloud_admin_password" {
|
||||
type = string
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "telegram_bot_token" {
|
||||
description = "Telegram bot token for Alertmanager notifications"
|
||||
type = string
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
@@ -4,3 +4,4 @@ TF_VAR_nextcloud_db_passwd=$MYSQL_NEXTCLOUD_PASSWORD
|
||||
TF_VAR_nextcloud_db_root_password=$MYSQL_ROOT_PASSWORD
|
||||
TF_VAR_nextcloud_admin_password=$NEXTCLOUD_ADMIN_PASSWORD
|
||||
TF_VAR_scaleway_project_id=$SCALEWAY_PROJECT_ID
|
||||
TF_VAR_telegram_bot_token=$TELEGRAM_BOT_TOKEN
|
||||
|
||||
@@ -132,8 +132,18 @@ module "proxy" {
|
||||
module "prometheus" {
|
||||
source = "../../modules/utils/prometheus"
|
||||
|
||||
cluster_name = "prod-01"
|
||||
persistent_folder = "${local.persistent_folder}/monitoring"
|
||||
config = file("${path.module}/resources/prometheus.yml")
|
||||
rules = local.prometheus_rules
|
||||
alertmanager_config = local.alertmanager_config
|
||||
|
||||
email_config = local.alertmanager_email_config
|
||||
telegram_config = {
|
||||
bot_token = var.telegram_bot_token
|
||||
chat_id = local.telegram_chat_id
|
||||
message = local.telegram_message
|
||||
}
|
||||
}
|
||||
|
||||
module "fail2ban_exporter" {
|
||||
|
||||
@@ -192,6 +192,21 @@ locals {
|
||||
}
|
||||
}
|
||||
|
||||
# prometheus
|
||||
prometheus_config = file("${path.module}/resources/prometheus.yml")
|
||||
prometheus_rules = file("${path.module}/resources/prometheus-rules.yml")
|
||||
alertmanager_config = file("${path.module}/resources/alertmanager.yml")
|
||||
alertmanager_email_config = {
|
||||
to = "c.alexandre.pires@alexpires.me"
|
||||
from = "alertmanager@${local.primary_domain}"
|
||||
smarthost = "smtp.tem.scw.cloud:587"
|
||||
auth_username = var.scaleway_project_id
|
||||
auth_password = local.mail_app_api_secret_key
|
||||
}
|
||||
|
||||
telegram_chat_id = "5432313610" # a13labs channel
|
||||
telegram_message = file("${path.module}/resources/telegram.tpl")
|
||||
|
||||
# get secrets from remote state
|
||||
backup_bucket_name = data.terraform_remote_state.scaleway.outputs.backup_bucket_name
|
||||
backup_app_api_secret_key = data.terraform_remote_state.scaleway.outputs.backup_app_api_secret_key
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
route:
|
||||
group_by: ["alertname"]
|
||||
receiver: "default"
|
||||
group_wait: 10s
|
||||
group_interval: 10m
|
||||
repeat_interval: 1h
|
||||
routes:
|
||||
- receiver: "telegram"
|
||||
matchers:
|
||||
- 'alertname="UserLoggedIn"'
|
||||
continue: false
|
||||
@@ -0,0 +1,10 @@
|
||||
groups:
|
||||
- name: auth.rules
|
||||
rules:
|
||||
- alert: UserLoggedIn
|
||||
expr: increase(auth_sessions_active_by_user[1m]) > 0
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: "Login: {{ $labels.username }} on {{ $labels.instance }}"
|
||||
description: "User {{ $labels.username }} logged in via {{ $labels.source }} on {{ $labels.instance }}."
|
||||
@@ -1,41 +1,4 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
rule_files:
|
||||
- /etc/prometheus/prometheus.rules
|
||||
scrape_configs:
|
||||
- job_name: kubernetes-nodes-cadvisor
|
||||
scrape_interval: 10s
|
||||
scrape_timeout: 10s
|
||||
scheme: https # remove if you want to scrape metrics on insecure port
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
insecure_skip_verify: true
|
||||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
relabel_configs:
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_node_label_(.+)
|
||||
# Only for Kubernetes ^1.7.3.
|
||||
# See: https://github.com/prometheus/prometheus/issues/2916
|
||||
- target_label: __address__
|
||||
replacement: kubernetes.default.svc:443
|
||||
- source_labels: [__meta_kubernetes_node_name]
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
|
||||
metric_relabel_configs:
|
||||
- action: replace
|
||||
source_labels: [id]
|
||||
regex: '^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$'
|
||||
target_label: rkt_container_name
|
||||
replacement: "${2}-${1}"
|
||||
- action: replace
|
||||
source_labels: [id]
|
||||
regex: '^/system\.slice/(.+)\.service$'
|
||||
target_label: systemd_service_name
|
||||
replacement: "${1}"
|
||||
- job_name: node-exporter
|
||||
scrape_interval: 10s
|
||||
static_configs:
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
{{/* Always produce a non-empty message. Handles single or multiple firing alerts and missing username. */}}
|
||||
{{ if gt (len .Alerts.Firing) 1 -}}
|
||||
<b>{{ len .Alerts.Firing }} logins</b> on {{ with .CommonLabels.cluster }}<b>{{ . }}</b>{{ else }}<b>unknown host</b>{{ end }}:
|
||||
{{- range .Alerts.Firing }}
|
||||
• {{ with .Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} — Source: <code>{{ with .Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with .Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
|
||||
{{- end }}
|
||||
{{ else if eq (len .Alerts.Firing) 1 -}}
|
||||
{{ $a := index .Alerts.Firing 0 -}}
|
||||
<b>Login:</b> {{ with $a.Labels.username }}<b>{{ . }}</b>{{ else }}<i>unknown</i>{{ end }} on <b>{{ $a.Labels.cluster }}</b>
|
||||
Source: <code>{{ with $a.Labels.source }}{{ . }}{{ else }}?{{ end }}</code>{{ with $a.Labels.method }}, Method: <code>{{ . }}</code>{{ else }}, Method: <code>?</code>{{ end }}
|
||||
{{ else -}}
|
||||
<b>Login detected</b> — details unavailable.
|
||||
{{ end -}}
|
||||
@@ -128,3 +128,9 @@ variable "nginx_token_bearer" {
|
||||
type = string
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
variable "telegram_bot_token" {
|
||||
description = "Telegram bot token for Alertmanager notifications"
|
||||
type = string
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
@@ -32,4 +32,4 @@ TF_VAR_wireguard_1_private_key=$WIREGUARD_1_PRIVATE_KEY
|
||||
TF_VAR_wireguard_1_public_key=$WIREGUARD_1_PUBLIC_KEY
|
||||
TF_VAR_wireguard_1_pre_shared_key=$WIREGUARD_1_PRE_SHARED_KEY
|
||||
TF_VAR_nginx_token_bearer=$NGINX_TOKEN_BEARER
|
||||
|
||||
TF_VAR_telegram_bot_token=$TELEGRAM_BOT_TOKEN
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
resource "kubernetes_config_map" "alertmanager" {
|
||||
metadata {
|
||||
name = "alertmanager-conf"
|
||||
namespace = kubernetes_namespace.prometheus.metadata[0].name
|
||||
labels = {
|
||||
name = "alertmanager-conf"
|
||||
}
|
||||
}
|
||||
|
||||
data = {
|
||||
"alertmanager.yml" = local.alertmanager_config
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_deployment" "alertmanager" {
|
||||
metadata {
|
||||
name = "alertmanager"
|
||||
namespace = kubernetes_namespace.prometheus.metadata[0].name
|
||||
labels = {
|
||||
app = "alertmanager"
|
||||
}
|
||||
}
|
||||
|
||||
spec {
|
||||
replicas = 1
|
||||
|
||||
selector {
|
||||
match_labels = {
|
||||
app = "alertmanager"
|
||||
}
|
||||
}
|
||||
|
||||
template {
|
||||
metadata {
|
||||
labels = {
|
||||
app = "alertmanager"
|
||||
}
|
||||
annotations = {
|
||||
"checksum/config" = local.alertmanager_config_checksum
|
||||
}
|
||||
}
|
||||
|
||||
spec {
|
||||
container {
|
||||
name = "alertmanager"
|
||||
image = "prom/alertmanager:${local.app_version}"
|
||||
|
||||
args = [
|
||||
"--config.file=/etc/alertmanager/alertmanager.yml",
|
||||
"--storage.path=/alertmanager"
|
||||
]
|
||||
|
||||
port {
|
||||
name = "http"
|
||||
container_port = 9093
|
||||
}
|
||||
|
||||
volume_mount {
|
||||
name = "alertmanager-config"
|
||||
mount_path = "/etc/alertmanager"
|
||||
}
|
||||
|
||||
volume_mount {
|
||||
name = "alertmanager-storage"
|
||||
mount_path = "/alertmanager"
|
||||
}
|
||||
|
||||
security_context {
|
||||
run_as_user = 1000
|
||||
run_as_group = 1000
|
||||
run_as_non_root = true
|
||||
allow_privilege_escalation = false
|
||||
}
|
||||
}
|
||||
|
||||
volume {
|
||||
name = "alertmanager-config"
|
||||
config_map {
|
||||
name = kubernetes_config_map.alertmanager.metadata[0].name
|
||||
}
|
||||
}
|
||||
|
||||
volume {
|
||||
name = "alertmanager-storage"
|
||||
host_path {
|
||||
path = local.alertmanager_folder
|
||||
type = "DirectoryOrCreate"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_service" "alertmanager" {
|
||||
metadata {
|
||||
name = "alertmanager"
|
||||
namespace = kubernetes_namespace.prometheus.metadata[0].name
|
||||
labels = {
|
||||
app = "alertmanager"
|
||||
}
|
||||
}
|
||||
|
||||
spec {
|
||||
selector = {
|
||||
app = "alertmanager"
|
||||
}
|
||||
|
||||
type = "ClusterIP"
|
||||
|
||||
port {
|
||||
name = "web"
|
||||
port = 9093
|
||||
protocol = "TCP"
|
||||
target_port = "http"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,17 +1,112 @@
|
||||
locals {
|
||||
app_version = var.tag
|
||||
data_folder = "${var.persistent_folder}/prometheus"
|
||||
prometheus_folder = "${var.persistent_folder}/prometheus"
|
||||
alertmanager_folder = "${var.persistent_folder}/alertmanager"
|
||||
|
||||
prometheus_config = coalesce(var.config, file("${path.module}/resources/default.yml"))
|
||||
prometheus_config_checksum = md5(local.prometheus_config)
|
||||
prometheus_default_config = yamldecode(file("${path.module}/resources/prometheus.yml"))
|
||||
alertmanager_default_config = yamldecode(file("${path.module}/resources/alertmanager.yml"))
|
||||
|
||||
prometheus_rules = yamlencode({
|
||||
groups = [
|
||||
prometheus_user_config = try(
|
||||
yamldecode(var.config),
|
||||
{}
|
||||
)
|
||||
prometheus_config = yamlencode({
|
||||
global = merge(
|
||||
try(local.prometheus_default_config.global, {}),
|
||||
try(local.prometheus_user_config.global, {}),
|
||||
{
|
||||
name = "custom-rules"
|
||||
rules = var.rules
|
||||
scrape_interval = "15s"
|
||||
evaluation_interval = "15s"
|
||||
external_labels = {
|
||||
cluster = var.cluster_name
|
||||
}
|
||||
}
|
||||
),
|
||||
rule_files = concat(
|
||||
try(local.prometheus_default_config.rule_files, []),
|
||||
try(local.prometheus_user_config.rule_files, [])
|
||||
),
|
||||
scrape_configs = concat(
|
||||
try(local.prometheus_default_config.scrape_configs, []),
|
||||
try(local.prometheus_user_config.scrape_configs, [])
|
||||
),
|
||||
alerting = merge(
|
||||
try(local.prometheus_default_config.alerting, {}),
|
||||
try(local.prometheus_user_config.alerting, {}),
|
||||
{
|
||||
alertmanagers = [
|
||||
{
|
||||
static_configs = [
|
||||
{
|
||||
targets = ["alertmanager.prometheus.svc.cluster.local:9093"]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
})
|
||||
prometheus_config_checksum = md5(local.prometheus_config)
|
||||
|
||||
prometheus_user_rules = try(
|
||||
yamldecode(var.rules),
|
||||
{}
|
||||
)
|
||||
prometheus_merged_rules = merge({ groups = [] }, local.prometheus_user_rules)
|
||||
prometheus_rules = yamlencode(local.prometheus_merged_rules)
|
||||
prometheus_rules_checksum = md5(local.prometheus_rules)
|
||||
|
||||
alertmanager_user_config = try(
|
||||
yamldecode(var.alertmanager_config),
|
||||
{}
|
||||
)
|
||||
alertmanager_config = yamlencode({
|
||||
global = merge(
|
||||
try(local.alertmanager_default_config.global, {}),
|
||||
try(local.alertmanager_user_config.global, {})
|
||||
),
|
||||
route = merge(
|
||||
try(local.alertmanager_default_config.route, {}),
|
||||
try(local.alertmanager_user_config.route, {
|
||||
receiver = "default"
|
||||
})
|
||||
),
|
||||
receivers = concat(
|
||||
try(local.alertmanager_default_config.receivers, []),
|
||||
try(local.alertmanager_user_config.receivers, []),
|
||||
[
|
||||
{
|
||||
name = "default"
|
||||
email_configs = [
|
||||
{
|
||||
to = var.email_config.to
|
||||
from = var.email_config.from
|
||||
smarthost = var.email_config.smarthost
|
||||
auth_username = var.email_config.auth_username
|
||||
auth_password = var.email_config.auth_password
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
try([
|
||||
{
|
||||
name = "telegram"
|
||||
telegram_configs = [
|
||||
{
|
||||
send_resolved = true
|
||||
parse_mode = var.telegram_config.parse_mode
|
||||
bot_token = var.telegram_config.bot_token
|
||||
chat_id = tonumber(var.telegram_config.chat_id)
|
||||
message = var.telegram_config.message != "" ? var.telegram_config.message : "{{ template \"telegram.default.message\" . }}"
|
||||
}
|
||||
]
|
||||
}
|
||||
], [])
|
||||
)
|
||||
templates = concat(
|
||||
try(local.prometheus_default_config.templates, []),
|
||||
try(local.prometheus_user_config.templates, [])
|
||||
)
|
||||
})
|
||||
alertmanager_config_checksum = md5(local.alertmanager_config)
|
||||
}
|
||||
|
||||
@@ -143,7 +143,7 @@ resource "kubernetes_deployment" "prometheus" {
|
||||
volume {
|
||||
name = "prometheus-storage-volume"
|
||||
host_path {
|
||||
path = local.data_folder
|
||||
path = local.prometheus_folder
|
||||
type = "DirectoryOrCreate"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
global:
|
||||
resolve_timeout: 5m
|
||||
route:
|
||||
receiver: "default"
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 12h
|
||||
@@ -1,126 +0,0 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
evaluation_interval: 5s
|
||||
rule_files:
|
||||
- /etc/prometheus/prometheus.rules
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- scheme: http
|
||||
static_configs:
|
||||
- targets:
|
||||
- "alertmanager.monitoring.svc:9093"
|
||||
scrape_configs:
|
||||
- job_name: "node-exporter"
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_endpoints_name]
|
||||
regex: "node-exporter"
|
||||
action: keep
|
||||
- job_name: "kubernetes-apiservers"
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
relabel_configs:
|
||||
- source_labels:
|
||||
[
|
||||
__meta_kubernetes_namespace,
|
||||
__meta_kubernetes_service_name,
|
||||
__meta_kubernetes_endpoint_port_name,
|
||||
]
|
||||
action: keep
|
||||
regex: default;kubernetes;https
|
||||
- job_name: "kubernetes-nodes"
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
relabel_configs:
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_node_label_(.+)
|
||||
- target_label: __address__
|
||||
replacement: kubernetes.default.svc:443
|
||||
- source_labels: [__meta_kubernetes_node_name]
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
replacement: /api/v1/nodes/$${1}/proxy/metrics
|
||||
- job_name: "kubernetes-pods"
|
||||
kubernetes_sd_configs:
|
||||
- role: pod
|
||||
relabel_configs:
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
|
||||
action: keep
|
||||
regex: true
|
||||
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
|
||||
action: replace
|
||||
target_label: __metrics_path__
|
||||
regex: (.+)
|
||||
- source_labels:
|
||||
[__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
|
||||
action: replace
|
||||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||
replacement: $1:$2
|
||||
target_label: __address__
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_pod_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
action: replace
|
||||
target_label: kubernetes_namespace
|
||||
- source_labels: [__meta_kubernetes_pod_name]
|
||||
action: replace
|
||||
target_label: kubernetes_pod_name
|
||||
- job_name: "kube-state-metrics"
|
||||
static_configs:
|
||||
- targets: ["kube-state-metrics.kube-system.svc.cluster.local:8080"]
|
||||
- job_name: "kubernetes-cadvisor"
|
||||
scheme: https
|
||||
tls_config:
|
||||
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
|
||||
kubernetes_sd_configs:
|
||||
- role: node
|
||||
relabel_configs:
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_node_label_(.+)
|
||||
- target_label: __address__
|
||||
replacement: kubernetes.default.svc:443
|
||||
- source_labels: [__meta_kubernetes_node_name]
|
||||
regex: (.+)
|
||||
target_label: __metrics_path__
|
||||
replacement: /api/v1/nodes/$${1}/proxy/metrics/cadvisor
|
||||
- job_name: "kubernetes-service-endpoints"
|
||||
kubernetes_sd_configs:
|
||||
- role: endpoints
|
||||
relabel_configs:
|
||||
- source_labels:
|
||||
[__meta_kubernetes_service_annotation_prometheus_io_scrape]
|
||||
action: keep
|
||||
regex: true
|
||||
- source_labels:
|
||||
[__meta_kubernetes_service_annotation_prometheus_io_scheme]
|
||||
action: replace
|
||||
target_label: __scheme__
|
||||
regex: (https?)
|
||||
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
|
||||
action: replace
|
||||
target_label: __metrics_path__
|
||||
regex: (.+)
|
||||
- source_labels:
|
||||
[__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
|
||||
action: replace
|
||||
target_label: __address__
|
||||
regex: ([^:]+)(?::\d+)?;(\d+)
|
||||
replacement: $1:$2
|
||||
- action: labelmap
|
||||
regex: __meta_kubernetes_service_label_(.+)
|
||||
- source_labels: [__meta_kubernetes_namespace]
|
||||
action: replace
|
||||
target_label: kubernetes_namespace
|
||||
- source_labels: [__meta_kubernetes_service_name]
|
||||
action: replace
|
||||
target_label: kubernetes_name
|
||||
+4
@@ -36,3 +36,7 @@ scrape_configs:
|
||||
regex: '^/system\.slice/(.+)\.service$'
|
||||
target_label: systemd_service_name
|
||||
replacement: "${1}"
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets: ["alertmanager.prometheus.svc.cluster.local:9093"]
|
||||
@@ -11,15 +11,9 @@ variable "retention_time" {
|
||||
}
|
||||
|
||||
variable "rules" {
|
||||
description = "Prometheus alerting rules"
|
||||
type = list(object({
|
||||
alert = string
|
||||
expr = string
|
||||
for = string
|
||||
labels = map(string)
|
||||
annotations = map(string)
|
||||
}))
|
||||
default = []
|
||||
description = "Prometheus alerting rules in YAML format"
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "config" {
|
||||
@@ -32,3 +26,44 @@ variable "persistent_folder" {
|
||||
description = "The path to the persistent folder"
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "alertmanager_config" {
|
||||
description = "Custom Alertmanager configuration in YAML format"
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "cluster_name" {
|
||||
description = "The name of the Kubernetes cluster"
|
||||
type = string
|
||||
default = "kubernetes"
|
||||
}
|
||||
|
||||
variable "email_config" {
|
||||
description = "Email configuration for Alertmanager in YAML format"
|
||||
type = object({
|
||||
to = optional(string, "root@localhost")
|
||||
from = optional(string, "root@localhost")
|
||||
smarthost = optional(string, "localhost:25")
|
||||
auth_username = optional(string, "")
|
||||
auth_password = optional(string, "")
|
||||
})
|
||||
default = {
|
||||
to = "root@localhost"
|
||||
from = "root@localhost"
|
||||
smarthost = "localhost:25"
|
||||
auth_username = ""
|
||||
auth_password = ""
|
||||
}
|
||||
}
|
||||
|
||||
variable "telegram_config" {
|
||||
description = "Telegram configuration for Alertmanager in YAML format"
|
||||
type = object({
|
||||
bot_token = string
|
||||
chat_id = string
|
||||
parse_mode = optional(string, "HTML")
|
||||
message = optional(string, "")
|
||||
})
|
||||
default = null
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user