Add GPU monitoring services and configurations
- Create systemd service templates for llama_exporter, nvidia_exporter, and podman. - Add Prometheus configuration template for GPU metrics scraping. - Introduce variables for GPU monitoring role in main.yml. - Implement tasks for syncing llama models, including user setup and package installation. - Update podman tasks to ensure proper sudo access and lingering for the podman user. - Modify inventory to include gpu_monitoring group. - Add Terraform modules for deploying Vaultwarden, including ingress and service configurations. - Create Grafana dashboard for real-time GPU monitoring metrics. - Update secrets and environment files to include Vaultwarden admin token.
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
---
|
||||
- name: Create amd_exporter group
|
||||
become: true
|
||||
ansible.builtin.group:
|
||||
name: "{{ amd_exporter_user }}"
|
||||
state: present
|
||||
system: true
|
||||
|
||||
- name: Create amd_exporter user
|
||||
become: true
|
||||
ansible.builtin.user:
|
||||
name: "{{ amd_exporter_user }}"
|
||||
group: "{{ amd_exporter_user }}"
|
||||
home: "{{ amd_exporter_user_home }}"
|
||||
create_home: true
|
||||
shell: /usr/sbin/nologin
|
||||
system: true
|
||||
|
||||
- name: Create amd_exporter install directory
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ amd_exporter_user }}"
|
||||
group: "{{ amd_exporter_user }}"
|
||||
mode: "0755"
|
||||
loop:
|
||||
- "{{ amd_exporter_install_dir }}"
|
||||
- "{{ amd_exporter_install_dir }}/logs"
|
||||
|
||||
- name: Copy amd_gpu_exporter script
|
||||
become: true
|
||||
ansible.builtin.copy:
|
||||
src: amd_gpu_exporter.py
|
||||
dest: "{{ amd_exporter_script_path }}"
|
||||
owner: "{{ amd_exporter_user }}"
|
||||
group: "{{ amd_exporter_user }}"
|
||||
mode: "0755"
|
||||
notify:
|
||||
- Restart amd_exporter
|
||||
|
||||
- name: Create Python virtual environment
|
||||
become: true
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ amd_exporter_python_version }} -m venv {{ amd_exporter_venv_path }}"
|
||||
creates: "{{ amd_exporter_venv_path }}/bin/activate"
|
||||
|
||||
- name: Install Python dependencies in venv
|
||||
become: true
|
||||
ansible.builtin.pip:
|
||||
name:
|
||||
- prometheus-client
|
||||
- requests
|
||||
virtualenv: "{{ amd_exporter_venv_path }}"
|
||||
state: present
|
||||
|
||||
- name: Fix ownership of amd_exporter directory
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "{{ amd_exporter_install_dir }}"
|
||||
owner: "{{ amd_exporter_user }}"
|
||||
group: "{{ amd_exporter_user }}"
|
||||
recurse: true
|
||||
|
||||
- name: Generate amd_exporter systemd service
|
||||
become: true
|
||||
ansible.builtin.template:
|
||||
src: amdgpu_exporter.service.j2
|
||||
dest: "/etc/systemd/system/{{ amd_exporter_service_name }}.service"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify:
|
||||
- Reload systemd daemon
|
||||
- Restart amd_exporter
|
||||
|
||||
- name: Enable amd_exporter service
|
||||
become: true
|
||||
ansible.builtin.systemd:
|
||||
name: "{{ amd_exporter_service_name }}"
|
||||
enabled: true
|
||||
state: started
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
- name: Add Prometheus firewall rules
|
||||
become: true
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
src: "{{ item.cidr }}"
|
||||
port: "{{ prometheus_fw_port | string }}"
|
||||
proto: "{{ prometheus_fw_proto }}"
|
||||
comment: "{{ item.comment | default(omit) }}"
|
||||
loop: "{{ prometheus_fw_allowed_networks }}"
|
||||
loop_control:
|
||||
label: "{{ item.cidr }} -> port {{ prometheus_fw_port }}"
|
||||
when: item.cidr is defined and item.cidr | length > 0
|
||||
@@ -0,0 +1,82 @@
|
||||
---
|
||||
- name: Create llama_exporter group
|
||||
become: true
|
||||
ansible.builtin.group:
|
||||
name: "{{ llama_exporter_user }}"
|
||||
state: present
|
||||
system: true
|
||||
|
||||
- name: Create llama_exporter user
|
||||
become: true
|
||||
ansible.builtin.user:
|
||||
name: "{{ llama_exporter_user }}"
|
||||
group: "{{ llama_exporter_user }}"
|
||||
home: "{{ llama_exporter_user_home }}"
|
||||
create_home: true
|
||||
shell: /usr/sbin/nologin
|
||||
system: true
|
||||
|
||||
- name: Create llama_exporter install directory
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ llama_exporter_user }}"
|
||||
group: "{{ llama_exporter_user }}"
|
||||
mode: "0755"
|
||||
loop:
|
||||
- "{{ llama_exporter_install_dir }}"
|
||||
- "{{ llama_exporter_install_dir }}/logs"
|
||||
|
||||
- name: Copy llama_exporter script
|
||||
become: true
|
||||
ansible.builtin.copy:
|
||||
src: llama_exporter.py
|
||||
dest: "{{ llama_exporter_script_path }}"
|
||||
owner: "{{ llama_exporter_user }}"
|
||||
group: "{{ llama_exporter_user }}"
|
||||
mode: "0755"
|
||||
notify:
|
||||
- Restart llama_exporter
|
||||
|
||||
- name: Create Python virtual environment
|
||||
become: true
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ llama_exporter_venv_path }}"
|
||||
creates: "{{ llama_exporter_venv_path }}/bin/activate"
|
||||
|
||||
- name: Install Python dependencies in llama_exporter venv
|
||||
become: true
|
||||
ansible.builtin.pip:
|
||||
name:
|
||||
- prometheus-client
|
||||
- requests
|
||||
virtualenv: "{{ llama_exporter_venv_path }}"
|
||||
state: present
|
||||
|
||||
- name: Fix ownership of llama_exporter directory
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "{{ llama_exporter_install_dir }}"
|
||||
owner: "{{ llama_exporter_user }}"
|
||||
group: "{{ llama_exporter_user }}"
|
||||
recurse: true
|
||||
|
||||
- name: Generate llama_exporter systemd service
|
||||
become: true
|
||||
ansible.builtin.template:
|
||||
src: llama_exporter.service.j2
|
||||
dest: "/etc/systemd/system/{{ llama_exporter_service_name }}.service"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
notify:
|
||||
- Reload systemd daemon
|
||||
- Restart llama_exporter
|
||||
|
||||
- name: Enable llama_exporter service
|
||||
become: true
|
||||
ansible.builtin.systemd:
|
||||
name: "{{ llama_exporter_service_name }}"
|
||||
enabled: true
|
||||
state: started
|
||||
@@ -0,0 +1,28 @@
|
||||
---
|
||||
- name: Include NVIDIA GPU Exporter setup
|
||||
when: nvidia_exporter_enabled | default(true) | bool
|
||||
ansible.builtin.include_tasks: nvidia_exporter.yml
|
||||
tags:
|
||||
- roles::gpu_monitoring::nvidia_exporter
|
||||
|
||||
- name: Include AMD GPU Exporter setup
|
||||
when: amd_exporter_enabled | default(true) | bool
|
||||
ansible.builtin.include_tasks: amd_exporter.yml
|
||||
tags:
|
||||
- roles::gpu_monitoring::amd_exporter
|
||||
|
||||
- name: Include Llama Exporter setup
|
||||
when: llama_exporter_enabled | default(true) | bool
|
||||
ansible.builtin.include_tasks: llama_exporter.yml
|
||||
tags:
|
||||
- roles::gpu_monitoring::llama_exporter
|
||||
|
||||
- name: Include Prometheus setup
|
||||
ansible.builtin.include_tasks: prometheus.yml
|
||||
tags:
|
||||
- roles::gpu_monitoring::prometheus
|
||||
|
||||
- name: Include firewall rules
|
||||
ansible.builtin.include_tasks: firewall.yml
|
||||
tags:
|
||||
- roles::gpu_monitoring::firewall
|
||||
@@ -0,0 +1,84 @@
|
||||
---
|
||||
# NVIDIA GPU Exporter using NVML (pynvml)
|
||||
# Works with consumer GPUs (RTX/GTX) and datacenter GPUs
|
||||
# No DCGM required - uses pynvml Python library
|
||||
|
||||
- name: Create NVIDIA exporter group
|
||||
ansible.builtin.group:
|
||||
name: "{{ nvidia_exporter_user }}"
|
||||
state: present
|
||||
system: true
|
||||
become: true
|
||||
|
||||
- name: Create NVIDIA exporter user
|
||||
ansible.builtin.user:
|
||||
name: "{{ nvidia_exporter_user }}"
|
||||
group: "{{ nvidia_exporter_user }}"
|
||||
home: "{{ nvidia_exporter_user_home }}"
|
||||
system: true
|
||||
create_home: true
|
||||
shell: /usr/sbin/nologin
|
||||
become: true
|
||||
|
||||
- name: Create NVIDIA exporter install directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ nvidia_exporter_user }}"
|
||||
group: "{{ nvidia_exporter_user }}"
|
||||
mode: "0755"
|
||||
loop:
|
||||
- "{{ nvidia_exporter_install_dir }}"
|
||||
- "{{ nvidia_exporter_install_dir }}/logs"
|
||||
become: true
|
||||
|
||||
- name: Copy NVIDIA exporter script
|
||||
ansible.builtin.copy:
|
||||
src: nvidia_gpu_exporter.py
|
||||
dest: "{{ nvidia_exporter_script_path }}"
|
||||
owner: "{{ nvidia_exporter_user }}"
|
||||
group: "{{ nvidia_exporter_user }}"
|
||||
mode: "0755"
|
||||
become: true
|
||||
notify:
|
||||
- Restart nvidia_exporter
|
||||
|
||||
- name: Create Python virtual environment for NVIDIA exporter
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ nvidia_exporter_venv_path }}"
|
||||
creates: "{{ nvidia_exporter_venv_path }}/bin/activate"
|
||||
become: true
|
||||
|
||||
- name: Install nvidia-ml-py in NVIDIA exporter venv
|
||||
ansible.builtin.pip:
|
||||
name: "nvidia-ml-py"
|
||||
virtualenv: "{{ nvidia_exporter_venv_path }}"
|
||||
state: present
|
||||
become: true
|
||||
|
||||
- name: Fix ownership of NVIDIA exporter directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ nvidia_exporter_install_dir }}"
|
||||
owner: "{{ nvidia_exporter_user }}"
|
||||
group: "{{ nvidia_exporter_user }}"
|
||||
recurse: true
|
||||
become: true
|
||||
|
||||
- name: Copy NVIDIA exporter systemd service
|
||||
ansible.builtin.template:
|
||||
src: nvidia_exporter.service.j2
|
||||
dest: "/etc/systemd/system/{{ nvidia_exporter_service_name }}.service"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
become: true
|
||||
notify:
|
||||
- Reload systemd daemon
|
||||
- Restart nvidia_exporter
|
||||
|
||||
- name: Ensure NVIDIA exporter is enabled and started
|
||||
ansible.builtin.systemd:
|
||||
name: "{{ nvidia_exporter_service_name }}"
|
||||
enabled: true
|
||||
state: started
|
||||
become: true
|
||||
@@ -0,0 +1,206 @@
|
||||
---
|
||||
# Prometheus - Podman container with standard podman.yml pattern
|
||||
# Follows the pattern in playbook_podman_llama_vulkan.yml
|
||||
|
||||
- name: Prometheus | Check if required vars are set
|
||||
ansible.builtin.fail:
|
||||
msg: "Required variables are not set for Podman setup. Please check your playbook variables.)"
|
||||
when: prometheus_user is not defined or prometheus_group is not defined or prometheus_user_home is not defined
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Create prometheus group
|
||||
become: true
|
||||
ansible.builtin.group:
|
||||
name: "{{ prometheus_group }}"
|
||||
system: true
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Create prometheus user
|
||||
become: true
|
||||
ansible.builtin.user:
|
||||
name: "{{ prometheus_user }}"
|
||||
groups: "{{ prometheus_extra_groups | default([]) }}"
|
||||
shell: /bin/bash
|
||||
home: "{{ prometheus_user_home }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Disable password login for prometheus user
|
||||
ansible.builtin.command: passwd -d "{{ prometheus_user }}"
|
||||
become: true
|
||||
changed_when: false
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Create prometheus user home
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "{{ prometheus_user_home }}"
|
||||
state: directory
|
||||
owner: "{{ prometheus_user }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
mode: "0750"
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Create build folders
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "{{ prometheus_user_home }}/build"
|
||||
state: directory
|
||||
owner: "{{ prometheus_user }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
mode: "0750"
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Create folders
|
||||
become: true
|
||||
ansible.builtin.file:
|
||||
path: "{{ prometheus_user_home }}/{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ prometheus_user }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
mode: "0750"
|
||||
loop: "{{ prometheus_user_folders | default([]) }}"
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Add SSH Authorized key
|
||||
become: true
|
||||
ansible.posix.authorized_key:
|
||||
user: "{{ prometheus_user }}"
|
||||
key: "{{ prometheus_user_pubkey }}"
|
||||
state: "{{ 'present' if prometheus_user_pubkey is defined else 'absent' }}"
|
||||
when: prometheus_user_pubkey is defined and prometheus_user_pubkey != ""
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | SELinux tasks (RedHat only)
|
||||
become: true
|
||||
when: ansible_os_family == 'RedHat'
|
||||
tags:
|
||||
- prometheus
|
||||
block:
|
||||
- name: Prometheus | Persistently set SELinux context
|
||||
community.general.sefcontext:
|
||||
target: "{{ prometheus_user_home }}(/.*)?"
|
||||
setype: user_home_dir_t
|
||||
state: present
|
||||
|
||||
- name: Prometheus | Persistently set SELinux context (ssh authorized keys)
|
||||
community.general.sefcontext:
|
||||
target: "{{ prometheus_user_home }}/.ssh/authorized_keys"
|
||||
setype: ssh_home_t
|
||||
state: present
|
||||
when: prometheus_user_pubkey is defined and prometheus_user_pubkey != ""
|
||||
|
||||
- name: Prometheus | Apply SELinux context
|
||||
ansible.builtin.command: restorecon -Rv {{ prometheus_user_home }}
|
||||
changed_when: false
|
||||
|
||||
- name: Prometheus | Enable lingering for prometheus user
|
||||
become: true
|
||||
ansible.builtin.command: loginctl enable-linger {{ prometheus_user }}
|
||||
register: linger_status
|
||||
changed_when: >-
|
||||
'created' in linger_status.stdout or
|
||||
(linger_status.rc == 0 and not
|
||||
('linger file already exists' in linger_status.stderr or
|
||||
'linger file does not exist' in linger_status.stderr))
|
||||
failed_when:
|
||||
- linger_status.rc != 0 and not
|
||||
('linger file already exists' in linger_status.stderr)
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Get prometheus user UID
|
||||
ansible.builtin.command: id -u {{ prometheus_user }}
|
||||
changed_when: false
|
||||
register: prometheus_uid
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Deploy Prometheus configuration
|
||||
ansible.builtin.template:
|
||||
src: prometheus.yml.j2
|
||||
dest: "{{ prometheus_user_home }}/prometheus.yml"
|
||||
owner: "{{ prometheus_user }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
mode: "0644"
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Create Prometheus data directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ prometheus_data_dir }}"
|
||||
state: directory
|
||||
owner: "{{ prometheus_user }}"
|
||||
group: "{{ prometheus_group }}"
|
||||
mode: "0700"
|
||||
tags:
|
||||
- prometheus
|
||||
|
||||
- name: Prometheus | Run Podman tasks as prometheus user
|
||||
become: true
|
||||
become_user: "{{ prometheus_user }}"
|
||||
environment:
|
||||
XDG_RUNTIME_DIR: "/run/user/{{ prometheus_uid.stdout }}"
|
||||
tags:
|
||||
- prometheus
|
||||
block:
|
||||
- name: Prometheus | Pull Prometheus image
|
||||
containers.podman.podman_image:
|
||||
name: "docker.io/prom/prometheus:{{ prometheus_image_tag }}"
|
||||
force: false
|
||||
|
||||
- name: Prometheus | Create podman network
|
||||
containers.podman.podman_network:
|
||||
name: "{{ prometheus_network_name }}"
|
||||
|
||||
- name: Prometheus | Create Podman container
|
||||
containers.podman.podman_container:
|
||||
name: prometheus
|
||||
image: "docker.io/prom/prometheus:{{ prometheus_image_tag }}"
|
||||
state: started
|
||||
recreate: true
|
||||
user: "{{ prometheus_uid.stdout }}"
|
||||
network: "{{ prometheus_network_name }}"
|
||||
network_alias:
|
||||
- prometheus
|
||||
ports:
|
||||
- "0.0.0.0:{{ prometheus_host_port }}:{{ prometheus_container_port }}/tcp"
|
||||
volumes:
|
||||
- "{{ prometheus_data_dir }}:/prometheus"
|
||||
- "{{ prometheus_user_home }}/prometheus.yml:/etc/prometheus/prometheus.yml"
|
||||
memory: "{{ prometheus_container_memory }}"
|
||||
memory_swap: "0"
|
||||
env:
|
||||
TZ: "{{ timezone | default('UTC') }}"
|
||||
PROMETHEUS_PORT: "{{ prometheus_port | string }}"
|
||||
|
||||
systemd: "true"
|
||||
restart: false
|
||||
detach: true
|
||||
label:
|
||||
a13labs.prometheus: "true"
|
||||
a13labs.service: "prometheus"
|
||||
a13labs.team: "infra"
|
||||
cmd_args:
|
||||
- "--userns=keep-id"
|
||||
- "--security-opt=label=disable"
|
||||
restart_policy: unless-stopped
|
||||
stop_signal: 1
|
||||
- name: Prometheus | Ensure systemd is reloaded after container creation
|
||||
become: true
|
||||
become_user: "{{ prometheus_user }}"
|
||||
environment:
|
||||
XDG_RUNTIME_DIR: "/run/user/{{ prometheus_uid.stdout }}"
|
||||
ansible.builtin.systemd:
|
||||
daemon_reload: true
|
||||
scope: user
|
||||
tags:
|
||||
- prometheus
|
||||
Reference in New Issue
Block a user