Add Node Exporter and Fail2Ban Exporter modules: implement Kubernetes resources and Prometheus scrape configurations

terraform/apps/dev-01/apps.tf

@@ -62,3 +62,7 @@ module "grafana" {
   persistent_folder = "${local.persistent_folder}/monitoring"
   fqdn              = local.grafana_fqdn
 }
+
+module "node_exporter" {
+  source = "../../modules/utils/node-exporter"
+}

terraform/apps/dev-01/resources/prometheus.yml

@@ -36,3 +36,9 @@ scrape_configs:
         regex: '^/system\.slice/(.+)\.service$'
         target_label: systemd_service_name
         replacement: "${1}"
+  - job_name: node-exporter
+    scrape_interval: 10s
+    static_configs:
+      - targets:
+          - "node-exporter.kube-system.svc.cluster.local:9100"
+          - "10.19.4.1:9100"

terraform/apps/prod-01/apps.tf

@@ -135,3 +135,11 @@ module "prometheus" {
   persistent_folder = "${local.persistent_folder}/monitoring"
   config            = file("${path.module}/resources/prometheus.yml")
 }
+
+module "fail2ban_exporter" {
+  source = "../../modules/utils/fail2ban-exporter"
+}
+
+module "node_exporter" {
+  source = "../../modules/utils/node-exporter"
+}

terraform/apps/prod-01/resources/prometheus.yml

@@ -36,5 +36,11 @@ scrape_configs:
         regex: '^/system\.slice/(.+)\.service$'
         target_label: systemd_service_name
         replacement: "${1}"
-remote_write:
+  - job_name: node-exporter
-  - url: "http://prometheus.reverse-proxy.svc.cluster.local:9090/api/v1/write"
+    scrape_interval: 10s
+    static_configs:
+      - targets: ["node-exporter.kube-system.svc.cluster.local:9100"]
+  - job_name: fail2ban
+    scrape_interval: 60s
+    static_configs:
+      - targets: ["fail2ban-exporter.kube-system.svc.cluster.local:9100"]

terraform/modules/utils/fail2ban-exporter/config.tf

@@ -1,3 +1,4 @@
 locals {
   app_version = var.tag
+  namespace   = var.namespace
 }

terraform/modules/utils/fail2ban-exporter/main.tf

@@ -0,0 +1,96 @@
+resource "kubernetes_service_account" "fail2ban_exporter" {
+  metadata {
+    name      = "fail2ban-exporter"
+    namespace = local.namespace
+    labels = {
+      k8s-app = "fail2ban-exporter"
+    }
+  }
+  automount_service_account_token = false
+}
+
+resource "kubernetes_deployment" "fail2ban_exporter" {
+  metadata {
+    name      = "fail2ban-exporter"
+    namespace = local.namespace
+    labels = {
+      k8s-app = "fail2ban-exporter"
+    }
+  }
+  spec {
+    selector {
+      match_labels = {
+        k8s-app = "fail2ban-exporter"
+      }
+    }
+    strategy {
+      type = "RollingUpdate"
+    }
+    template {
+      metadata {
+        labels = {
+          k8s-app = "fail2ban-exporter"
+        }
+      }
+      spec {
+        service_account_name            = "fail2ban-exporter"
+        automount_service_account_token = false
+        container {
+          name              = "fail2ban-exporter"
+          image             = "registry.gitlab.com/hctrdev/fail2ban-prometheus-exporter:${local.app_version}"
+          image_pull_policy = "IfNotPresent"
+          port {
+            container_port = 9191
+            name           = "http"
+            protocol       = "TCP"
+          }
+          security_context {
+            read_only_root_filesystem = true
+          }
+          volume_mount {
+            name       = "tmp-dir"
+            mount_path = "/tmp"
+          }
+          volume_mount {
+            name       = "fail2ban-run"
+            mount_path = "/var/run/fail2ban"
+            read_only  = true
+          }
+        }
+        volume {
+          name = "tmp-dir"
+          empty_dir {}
+        }
+        volume {
+          name = "fail2ban-run"
+          host_path {
+            path = "/var/run/fail2ban"
+            type = "Directory"
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service" "fail2ban_exporter" {
+  metadata {
+    name      = "fail2ban-exporter"
+    namespace = local.namespace
+    labels = {
+      k8s-app = "fail2ban-exporter"
+    }
+  }
+  spec {
+    selector = {
+      k8s-app = "fail2ban-exporter"
+    }
+    port {
+      name        = "http"
+      port        = 9100
+      target_port = 9191
+      protocol    = "TCP"
+    }
+    type = "ClusterIP"
+  }
+}

terraform/modules/utils/fail2ban-exporter/variables.tf

@@ -0,0 +1,11 @@
+variable "tag" {
+  description = "Fail2Ban exporter image tag"
+  type        = string
+  default     = "latest"
+}
+
+variable "namespace" {
+  description = "Kubernetes namespace to deploy the Fail2Ban exporter"
+  type        = string
+  default     = "kube-system"
+}

terraform/modules/utils/metrics_server/main.tf

@@ -1,259 +0,0 @@
-
-resource "kubernetes_service_account" "metrics_server" {
-  metadata {
-    name      = "metrics-server"
-    namespace = var.namespace
-    labels = {
-      k8s-app = "metrics-server"
-    }
-  }
-}
-
-resource "kubernetes_cluster_role" "aggregated_metrics_reader" {
-  metadata {
-    name = "system:aggregated-metrics-reader"
-    labels = {
-      k8s-app                                        = "metrics-server"
-      "rbac.authorization.k8s.io/aggregate-to-admin" = "true"
-      "rbac.authorization.k8s.io/aggregate-to-edit"  = "true"
-      "rbac.authorization.k8s.io/aggregate-to-view"  = "true"
-    }
-  }
-  rule {
-    api_groups = ["metrics.k8s.io"]
-    resources  = ["pods", "nodes"]
-    verbs      = ["get", "list", "watch"]
-  }
-}
-
-resource "kubernetes_cluster_role" "metrics_server" {
-  metadata {
-    name = "system:metrics-server"
-    labels = {
-      k8s-app = "metrics-server"
-    }
-  }
-  rule {
-    api_groups = [""]
-    resources  = ["nodes/metrics"]
-    verbs      = ["get"]
-  }
-  rule {
-    api_groups = [""]
-    resources  = ["pods", "nodes"]
-    verbs      = ["get", "list", "watch"]
-  }
-}
-
-resource "kubernetes_role_binding" "metrics_server_auth_reader" {
-  metadata {
-    name      = "metrics-server-auth-reader"
-    namespace = var.namespace
-    labels = {
-      k8s-app = "metrics-server"
-    }
-  }
-  role_ref {
-    api_group = "rbac.authorization.k8s.io"
-    kind      = "Role"
-    name      = "extension-apiserver-authentication-reader"
-  }
-  subject {
-    kind      = "ServiceAccount"
-    name      = "metrics-server"
-    namespace = var.namespace
-  }
-}
-
-resource "kubernetes_cluster_role_binding" "metrics_server_auth_delegator" {
-  metadata {
-    name = "metrics-server:system:auth-delegator"
-    labels = {
-      k8s-app = "metrics-server"
-    }
-  }
-  role_ref {
-    api_group = "rbac.authorization.k8s.io"
-    kind      = "ClusterRole"
-    name      = "system:auth-delegator"
-  }
-  subject {
-    kind      = "ServiceAccount"
-    name      = "metrics-server"
-    namespace = var.namespace
-  }
-}
-
-resource "kubernetes_cluster_role_binding" "metrics_server" {
-  metadata {
-    name = "system:metrics-server"
-    labels = {
-      k8s-app = "metrics-server"
-    }
-  }
-  role_ref {
-    api_group = "rbac.authorization.k8s.io"
-    kind      = "ClusterRole"
-    name      = "system:metrics-server"
-  }
-  subject {
-    kind      = "ServiceAccount"
-    name      = "metrics-server"
-    namespace = var.namespace
-  }
-}
-
-resource "kubernetes_service" "metrics_server" {
-  metadata {
-    name      = "metrics-server"
-    namespace = var.namespace
-    labels = {
-      k8s-app = "metrics-server"
-    }
-  }
-  spec {
-    selector = {
-      k8s-app = "metrics-server"
-    }
-    port {
-      name        = "https"
-      port        = 443
-      protocol    = "TCP"
-      target_port = "https"
-    }
-  }
-}
-
-resource "kubernetes_deployment" "metrics_server" {
-  metadata {
-    name      = "metrics-server"
-    namespace = var.namespace
-    labels = {
-      k8s-app = "metrics-server"
-    }
-  }
-  spec {
-    selector {
-      match_labels = {
-        k8s-app = "metrics-server"
-      }
-    }
-    strategy {
-      type = "RollingUpdate"
-      rolling_update {
-        max_unavailable = 0
-      }
-    }
-    template {
-      metadata {
-        labels = {
-          k8s-app = "metrics-server"
-        }
-      }
-      spec {
-        service_account_name = "metrics-server"
-        priority_class_name  = "system-cluster-critical"
-        node_selector = {
-          "kubernetes.io/os"   = "linux"
-          "kubernetes.io/arch" = "$ARCH"
-        }
-        container {
-          name              = "metrics-server"
-          image             = "registry.k8s.io/metrics-server/metrics-server:${local.app_version}"
-          image_pull_policy = "IfNotPresent"
-          args = [
-            "--cert-dir=/tmp",
-            "--secure-port=4443",
-            "--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname",
-            "--kubelet-use-node-status-port",
-            "--metric-resolution=15s",
-            "--kubelet-insecure-tls"
-          ]
-          port {
-            container_port = 4443
-            name           = "https"
-            protocol       = "TCP"
-          }
-          liveness_probe {
-            http_get {
-              path   = "/livez"
-              port   = "https"
-              scheme = "HTTPS"
-            }
-            failure_threshold = 3
-            period_seconds    = 10
-          }
-          readiness_probe {
-            http_get {
-              path   = "/readyz"
-              port   = "https"
-              scheme = "HTTPS"
-            }
-            failure_threshold     = 3
-            initial_delay_seconds = 20
-            period_seconds        = 10
-          }
-          resources {
-            requests = {
-              cpu    = "100m"
-              memory = "200Mi"
-            }
-          }
-          security_context {
-            read_only_root_filesystem = true
-            run_as_non_root           = true
-            run_as_user               = 1000
-          }
-          volume_mount {
-            name       = "tmp-dir"
-            mount_path = "/tmp"
-          }
-        }
-        volume {
-          name = "tmp-dir"
-          empty_dir {}
-        }
-      }
-    }
-  }
-}
-
-resource "kubernetes_manifest" "metrics_server_apiservice" {
-  manifest = {
-    apiVersion = "apiregistration.k8s.io/v1"
-    kind       = "APIService"
-    metadata = {
-      name = "v1beta1.metrics.k8s.io"
-      labels = {
-        k8s-app = "metrics-server"
-      }
-    }
-    spec = {
-      group                 = "metrics.k8s.io"
-      groupPriorityMinimum  = 100
-      insecureSkipTLSVerify = true
-      service = {
-        name      = "metrics-server"
-        namespace = var.namespace
-      }
-      version         = "v1beta1"
-      versionPriority = 100
-    }
-  }
-}
-
-# resource "kubernetes_cluster_role_binding" "microk8s_admin" {
-#   metadata {
-#     name = "microk8s-admin"
-#   }
-#   role_ref {
-#     api_group = "rbac.authorization.k8s.io"
-#     kind      = "ClusterRole"
-#     name      = "admin"
-#   }
-#   subject {
-#     kind      = "User"
-#     name      = "front-proxy-client"
-#     api_group = "rbac.authorization.k8s.io"
-#   }
-# }

terraform/modules/utils/metrics_server/variables.tf

@@ -1,11 +0,0 @@
-variable "tag" {
-  description = "Prometheus image tag"
-  type        = string
-  default     = "v0.8.0"
-}
-
-variable "namespace" {
-  description = "Kubernetes namespace to deploy the metrics server"
-  type        = string
-  default     = "kube-system"
-}

terraform/modules/utils/node-exporter/config.tf

@@ -0,0 +1,4 @@
+locals {
+  app_version = var.tag
+  namespace   = var.namespace
+}

terraform/modules/utils/node-exporter/main.tf

@@ -0,0 +1,97 @@
+resource "kubernetes_service_account" "node_exporter" {
+  metadata {
+    name      = "node-exporter"
+    namespace = local.namespace
+    labels = {
+      k8s-app = "node-exporter"
+    }
+  }
+  automount_service_account_token = false
+}
+
+resource "kubernetes_deployment" "node_exporter" {
+  metadata {
+    name      = "node-exporter"
+    namespace = local.namespace
+    labels = {
+      k8s-app = "node-exporter"
+    }
+  }
+  spec {
+    selector {
+      match_labels = {
+        k8s-app = "node-exporter"
+      }
+    }
+    strategy {
+      type = "RollingUpdate"
+    }
+    template {
+      metadata {
+        labels = {
+          k8s-app = "node-exporter"
+        }
+      }
+      spec {
+        service_account_name            = "node-exporter"
+        automount_service_account_token = false
+        container {
+          name              = "node-exporter"
+          image             = "quay.io/prometheus/node-exporter:${local.app_version}"
+          image_pull_policy = "IfNotPresent"
+          args              = ["--path.rootfs=/host/root"]
+          port {
+            container_port = 9100
+            name           = "http"
+            protocol       = "TCP"
+          }
+          security_context {
+            read_only_root_filesystem = true
+          }
+          volume_mount {
+            name       = "tmp-dir"
+            mount_path = "/tmp"
+          }
+          volume_mount {
+            name       = "rootfs"
+            mount_path = "/host/root"
+            read_only  = true
+          }
+        }
+        volume {
+          name = "tmp-dir"
+          empty_dir {}
+        }
+        volume {
+          name = "rootfs"
+          host_path {
+            path = "/"
+            type = "Directory"
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service" "node_exporter" {
+  metadata {
+    name      = "node-exporter"
+    namespace = local.namespace
+    labels = {
+      k8s-app = "node-exporter"
+    }
+  }
+  spec {
+    selector = {
+      k8s-app = "node-exporter"
+    }
+    port {
+      name        = "http"
+      port        = 9100
+      target_port = 9100
+      protocol    = "TCP"
+    }
+    type = "ClusterIP"
+  }
+}

terraform/modules/utils/node-exporter/provider.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = "~>1.8"
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.36.0"
+    }
+  }
+}

terraform/modules/utils/node-exporter/variables.tf

@@ -0,0 +1,11 @@
+variable "tag" {
+  description = "Fail2Ban exporter image tag"
+  type        = string
+  default     = "latest"
+}
+
+variable "namespace" {
+  description = "Kubernetes namespace to deploy the Fail2Ban exporter"
+  type        = string
+  default     = "kube-system"
+}