a13labs.infra/ansible/roles/gpu_monitoring/tasks/prometheus.yml

---
# Prometheus - Podman container with standard podman.yml pattern
# Follows the pattern in playbook_podman_llama_vulkan.yml

- name: Prometheus | Check if required vars are set
  ansible.builtin.fail:
    msg: "Required variables are not set for Podman setup. Please check your playbook variables.)"
  when: prometheus_user is not defined or prometheus_group is not defined or prometheus_user_home is not defined
  tags:
    - prometheus

- name: Prometheus | Create prometheus group
  become: true
  ansible.builtin.group:
    name: "{{ prometheus_group }}"
    system: true
  tags:
    - prometheus

- name: Prometheus | Create prometheus user
  become: true
  ansible.builtin.user:
    name: "{{ prometheus_user }}"
    groups: "{{ prometheus_extra_groups | default([]) }}"
    shell: /bin/bash
    home: "{{ prometheus_user_home }}"
    group: "{{ prometheus_group }}"
  tags:
    - prometheus

- name: Prometheus | Disable password login for prometheus user
  ansible.builtin.command: passwd -d "{{ prometheus_user }}"
  become: true
  changed_when: false
  tags:
    - prometheus

- name: Prometheus | Create prometheus user home
  become: true
  ansible.builtin.file:
    path: "{{ prometheus_user_home }}"
    state: directory
    owner: "{{ prometheus_user }}"
    group: "{{ prometheus_group }}"
    mode: "0750"
  tags:
    - prometheus

- name: Prometheus | Create build folders
  become: true
  ansible.builtin.file:
    path: "{{ prometheus_user_home }}/build"
    state: directory
    owner: "{{ prometheus_user }}"
    group: "{{ prometheus_group }}"
    mode: "0750"
  tags:
    - prometheus

- name: Prometheus | Create folders
  become: true
  ansible.builtin.file:
    path: "{{ prometheus_user_home }}/{{ item }}"
    state: directory
    owner: "{{ prometheus_user }}"
    group: "{{ prometheus_group }}"
    mode: "0750"
  loop: "{{ prometheus_user_folders | default([]) }}"
  tags:
    - prometheus

- name: Prometheus | Add SSH Authorized key
  become: true
  ansible.posix.authorized_key:
    user: "{{ prometheus_user }}"
    key: "{{ prometheus_user_pubkey }}"
    state: "{{ 'present' if prometheus_user_pubkey is defined else 'absent' }}"
  when: prometheus_user_pubkey is defined and prometheus_user_pubkey != ""
  tags:
    - prometheus

- name: Prometheus | SELinux tasks (RedHat only)
  become: true
  when: ansible_os_family == 'RedHat'
  tags:
    - prometheus
  block:
    - name: Prometheus | Persistently set SELinux context
      community.general.sefcontext:
        target: "{{ prometheus_user_home }}(/.*)?"
        setype: user_home_dir_t
        state: present

    - name: Prometheus | Persistently set SELinux context (ssh authorized keys)
      community.general.sefcontext:
        target: "{{ prometheus_user_home }}/.ssh/authorized_keys"
        setype: ssh_home_t
        state: present
      when: prometheus_user_pubkey is defined and prometheus_user_pubkey != ""

    - name: Prometheus | Apply SELinux context
      ansible.builtin.command: restorecon -Rv {{ prometheus_user_home }}
      changed_when: false

- name: Prometheus | Enable lingering for prometheus user
  become: true
  ansible.builtin.command: loginctl enable-linger {{ prometheus_user }}
  register: linger_status
  changed_when: >-
    'created' in linger_status.stdout or
    (linger_status.rc == 0 and not
      ('linger file already exists' in linger_status.stderr or
      'linger file does not exist' in linger_status.stderr))
  failed_when:
    - linger_status.rc != 0 and not
      ('linger file already exists' in linger_status.stderr)
  tags:
    - prometheus

- name: Prometheus | Get prometheus user UID
  ansible.builtin.command: id -u {{ prometheus_user }}
  changed_when: false
  register: prometheus_uid
  tags:
    - prometheus

- name: Prometheus | Deploy Prometheus configuration
  ansible.builtin.template:
    src: prometheus.yml.j2
    dest: "{{ prometheus_user_home }}/prometheus.yml"
    owner: "{{ prometheus_user }}"
    group: "{{ prometheus_group }}"
    mode: "0644"
  tags:
    - prometheus

- name: Prometheus | Create Prometheus data directory
  ansible.builtin.file:
    path: "{{ prometheus_data_dir }}"
    state: directory
    owner: "{{ prometheus_user }}"
    group: "{{ prometheus_group }}"
    mode: "0700"
  tags:
    - prometheus

- name: Prometheus | Run Podman tasks as prometheus user
  become: true
  become_user: "{{ prometheus_user }}"
  environment:
    XDG_RUNTIME_DIR: "/run/user/{{ prometheus_uid.stdout }}"
  tags:
    - prometheus
  block:
    - name: Prometheus | Pull Prometheus image
      containers.podman.podman_image:
        name: "docker.io/prom/prometheus:{{ prometheus_image_tag }}"
        force: false

    - name: Prometheus | Create podman network
      containers.podman.podman_network:
        name: "{{ prometheus_network_name }}"

    - name: Prometheus | Create Podman container
      containers.podman.podman_container:
        name: prometheus
        image: "docker.io/prom/prometheus:{{ prometheus_image_tag }}"
        state: started
        recreate: true
        user: "{{ prometheus_uid.stdout }}"
        network: "{{ prometheus_network_name }}"
        network_alias:
          - prometheus
        ports:
          - "0.0.0.0:{{ prometheus_host_port }}:{{ prometheus_container_port }}/tcp"
        volumes:
          - "{{ prometheus_data_dir }}:/prometheus"
          - "{{ prometheus_user_home }}/prometheus.yml:/etc/prometheus/prometheus.yml"
        memory: "{{ prometheus_container_memory }}"
        memory_swap: "0"
        env:
          TZ: "{{ timezone | default('UTC') }}"
          PROMETHEUS_PORT: "{{ prometheus_port | string }}"

        systemd: "true"
        restart: false
        detach: true
        label:
          a13labs.prometheus: "true"
          a13labs.service: "prometheus"
          a13labs.team: "infra"
        cmd_args:
          - "--userns=keep-id"
          - "--security-opt=label=disable"
        restart_policy: unless-stopped
        stop_signal: 1
- name: Prometheus | Ensure systemd is reloaded after container creation
  become: true
  become_user: "{{ prometheus_user }}"
  environment:
    XDG_RUNTIME_DIR: "/run/user/{{ prometheus_uid.stdout }}"
  ansible.builtin.systemd:
    daemon_reload: true
    scope: user
  tags:
    - prometheus