Add GPU monitoring services and configurations
- Create systemd service templates for llama_exporter, nvidia_exporter, and podman. - Add Prometheus configuration template for GPU metrics scraping. - Introduce variables for GPU monitoring role in main.yml. - Implement tasks for syncing llama models, including user setup and package installation. - Update podman tasks to ensure proper sudo access and lingering for the podman user. - Modify inventory to include gpu_monitoring group. - Add Terraform modules for deploying Vaultwarden, including ingress and service configurations. - Create Grafana dashboard for real-time GPU monitoring metrics. - Update secrets and environment files to include Vaultwarden admin token.
This commit is contained in:
@@ -0,0 +1,206 @@
|
||||
---
|
||||
# Prometheus - Podman container with standard podman.yml pattern
|
||||
# Follows the pattern in playbook_podman_llama_vulkan.yml
|
||||
|
||||
- name: Prometheus | Check if required vars are set
|
||||
ansible.builtin.fail:
|
||||
msg: "Required variables are not set for Podman setup. Please check your playbook variables.)"
|
||||
when: prometheus_user is not defined or prometheus_group is not defined or prometheus_user_home is not defined
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Create prometheus group
|
||||
become: true
|
||||
ansible.builtin.group:
|
||||
name: "{{ prometheus_group }}"
|
||||
system: true
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Create prometheus user
|
||||
become: true
|
||||
ansible.builtin.user:
|
||||
name: "{{ prometheus_user }}"
|
||||
groups: "{{ prometheus_extra_groups | default([]) }}"
|
||||
shell: /bin/bash
|
||||
home: "{{ prometheus_user_home }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Disable password login for prometheus user
|
||||
ansible.builtin.command: passwd -d "{{ prometheus_user }}"
|
||||
become: true
|
||||
changed_when: false
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Create prometheus user home
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "{{ prometheus_user_home }}"
|
||||
state: directory
|
||||
owner: "{{ prometheus_user }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
mode: "0750"
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Create build folders
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "{{ prometheus_user_home }}/build"
|
||||
state: directory
|
||||
owner: "{{ prometheus_user }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
mode: "0750"
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Create folders
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "{{ prometheus_user_home }}/{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ prometheus_user }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
mode: "0750"
|
||||
loop: "{{ prometheus_user_folders | default([]) }}"
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Add SSH Authorized key
|
||||
become: true
|
||||
ansible.posix.authorized_key:
|
||||
user: "{{ prometheus_user }}"
|
||||
key: "{{ prometheus_user_pubkey }}"
|
||||
state: "{{ 'present' if prometheus_user_pubkey is defined else 'absent' }}"
|
||||
when: prometheus_user_pubkey is defined and prometheus_user_pubkey != ""
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | SELinux tasks (RedHat only)
|
||||
become: true
|
||||
when: ansible_os_family == 'RedHat'
|
||||
tags:
|
||||
- prometheus
|
||||
block:
|
||||
- name: Prometheus | Persistently set SELinux context
|
||||
community.general.sefcontext:
|
||||
target: "{{ prometheus_user_home }}(/.*)?"
|
||||
setype: user_home_dir_t
|
||||
state: present
|
||||
|
||||
- name: Prometheus | Persistently set SELinux context (ssh authorized keys)
|
||||
community.general.sefcontext:
|
||||
target: "{{ prometheus_user_home }}/.ssh/authorized_keys"
|
||||
setype: ssh_home_t
|
||||
state: present
|
||||
when: prometheus_user_pubkey is defined and prometheus_user_pubkey != ""
|
||||
|
||||
- name: Prometheus | Apply SELinux context
|
||||
ansible.builtin.command: restorecon -Rv {{ prometheus_user_home }}
|
||||
changed_when: false
|
||||
|
||||
- name: Prometheus | Enable lingering for prometheus user
|
||||
become: true
|
||||
ansible.builtin.command: loginctl enable-linger {{ prometheus_user }}
|
||||
register: linger_status
|
||||
changed_when: >-
|
||||
'created' in linger_status.stdout or
|
||||
(linger_status.rc == 0 and not
|
||||
('linger file already exists' in linger_status.stderr or
|
||||
'linger file does not exist' in linger_status.stderr))
|
||||
failed_when:
|
||||
- linger_status.rc != 0 and not
|
||||
('linger file already exists' in linger_status.stderr)
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Get prometheus user UID
|
||||
ansible.builtin.command: id -u {{ prometheus_user }}
|
||||
changed_when: false
|
||||
register: prometheus_uid
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Deploy Prometheus configuration
|
||||
ansible.builtin.template:
|
||||
src: prometheus.yml.j2
|
||||
dest: "{{ prometheus_user_home }}/prometheus.yml"
|
||||
owner: "{{ prometheus_user }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
mode: "0644"
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Create Prometheus data directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ prometheus_data_dir }}"
|
||||
state: directory
|
||||
owner: "{{ prometheus_user }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
mode: "0700"
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Run Podman tasks as prometheus user
|
||||
become: true
|
||||
become_user: "{{ prometheus_user }}"
|
||||
environment:
|
||||
XDG_RUNTIME_DIR: "/run/user/{{ prometheus_uid.stdout }}"
|
||||
tags:
|
||||
- prometheus
|
||||
block:
|
||||
- name: Prometheus | Pull Prometheus image
|
||||
containers.podman.podman_image:
|
||||
name: "docker.io/prom/prometheus:{{ prometheus_image_tag }}"
|
||||
force: false
|
||||
|
||||
- name: Prometheus | Create podman network
|
||||
containers.podman.podman_network:
|
||||
name: "{{ prometheus_network_name }}"
|
||||
|
||||
- name: Prometheus | Create Podman container
|
||||
containers.podman.podman_container:
|
||||
name: prometheus
|
||||
image: "docker.io/prom/prometheus:{{ prometheus_image_tag }}"
|
||||
state: started
|
||||
recreate: true
|
||||
user: "{{ prometheus_uid.stdout }}"
|
||||
network: "{{ prometheus_network_name }}"
|
||||
network_alias:
|
||||
- prometheus
|
||||
ports:
|
||||
- "0.0.0.0:{{ prometheus_host_port }}:{{ prometheus_container_port }}/tcp"
|
||||
volumes:
|
||||
- "{{ prometheus_data_dir }}:/prometheus"
|
||||
- "{{ prometheus_user_home }}/prometheus.yml:/etc/prometheus/prometheus.yml"
|
||||
memory: "{{ prometheus_container_memory }}"
|
||||
memory_swap: "0"
|
||||
env:
|
||||
TZ: "{{ timezone | default('UTC') }}"
|
||||
PROMETHEUS_PORT: "{{ prometheus_port | string }}"
|
||||
|
||||
systemd: "true"
|
||||
restart: false
|
||||
detach: true
|
||||
label:
|
||||
a13labs.prometheus: "true"
|
||||
a13labs.service: "prometheus"
|
||||
a13labs.team: "infra"
|
||||
cmd_args:
|
||||
- "--userns=keep-id"
|
||||
- "--security-opt=label=disable"
|
||||
restart_policy: unless-stopped
|
||||
stop_signal: 1
|
||||
- name: Prometheus | Ensure systemd is reloaded after container creation
|
||||
become: true
|
||||
become_user: "{{ prometheus_user }}"
|
||||
environment:
|
||||
XDG_RUNTIME_DIR: "/run/user/{{ prometheus_uid.stdout }}"
|
||||
ansible.builtin.systemd:
|
||||
daemon_reload: true
|
||||
scope: user
|
||||
tags:
|
||||
- prometheus
|
||||
Reference in New Issue
Block a user