Add GPU monitoring services and configurations
- Create systemd service templates for llama_exporter, nvidia_exporter, and podman. - Add Prometheus configuration template for GPU metrics scraping. - Introduce variables for GPU monitoring role in main.yml. - Implement tasks for syncing llama models, including user setup and package installation. - Update podman tasks to ensure proper sudo access and lingering for the podman user. - Modify inventory to include gpu_monitoring group. - Add Terraform modules for deploying Vaultwarden, including ingress and service configurations. - Create Grafana dashboard for real-time GPU monitoring metrics. - Update secrets and environment files to include Vaultwarden admin token.
This commit is contained in:
@@ -0,0 +1,84 @@
|
||||
---
|
||||
# NVIDIA GPU Exporter using NVML (pynvml)
|
||||
# Works with consumer GPUs (RTX/GTX) and datacenter GPUs
|
||||
# No DCGM required - uses pynvml Python library
|
||||
|
||||
- name: Create NVIDIA exporter group
|
||||
ansible.builtin.group:
|
||||
name: "{{ nvidia_exporter_user }}"
|
||||
state: present
|
||||
system: true
|
||||
become: true
|
||||
|
||||
- name: Create NVIDIA exporter user
|
||||
ansible.builtin.user:
|
||||
name: "{{ nvidia_exporter_user }}"
|
||||
group: "{{ nvidia_exporter_user }}"
|
||||
home: "{{ nvidia_exporter_user_home }}"
|
||||
system: true
|
||||
create_home: true
|
||||
shell: /usr/sbin/nologin
|
||||
become: true
|
||||
|
||||
- name: Create NVIDIA exporter install directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ nvidia_exporter_user }}"
|
||||
group: "{{ nvidia_exporter_user }}"
|
||||
mode: "0755"
|
||||
loop:
|
||||
- "{{ nvidia_exporter_install_dir }}"
|
||||
- "{{ nvidia_exporter_install_dir }}/logs"
|
||||
become: true
|
||||
|
||||
- name: Copy NVIDIA exporter script
|
||||
ansible.builtin.copy:
|
||||
src: nvidia_gpu_exporter.py
|
||||
dest: "{{ nvidia_exporter_script_path }}"
|
||||
owner: "{{ nvidia_exporter_user }}"
|
||||
group: "{{ nvidia_exporter_user }}"
|
||||
mode: "0755"
|
||||
become: true
|
||||
notify:
|
||||
- Restart nvidia_exporter
|
||||
|
||||
- name: Create Python virtual environment for NVIDIA exporter
|
||||
ansible.builtin.command:
|
||||
cmd: "{{ ansible_facts['python']['executable'] }} -m venv {{ nvidia_exporter_venv_path }}"
|
||||
creates: "{{ nvidia_exporter_venv_path }}/bin/activate"
|
||||
become: true
|
||||
|
||||
- name: Install nvidia-ml-py in NVIDIA exporter venv
|
||||
ansible.builtin.pip:
|
||||
name: "nvidia-ml-py"
|
||||
virtualenv: "{{ nvidia_exporter_venv_path }}"
|
||||
state: present
|
||||
become: true
|
||||
|
||||
- name: Fix ownership of NVIDIA exporter directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ nvidia_exporter_install_dir }}"
|
||||
owner: "{{ nvidia_exporter_user }}"
|
||||
group: "{{ nvidia_exporter_user }}"
|
||||
recurse: true
|
||||
become: true
|
||||
|
||||
- name: Copy NVIDIA exporter systemd service
|
||||
ansible.builtin.template:
|
||||
src: nvidia_exporter.service.j2
|
||||
dest: "/etc/systemd/system/{{ nvidia_exporter_service_name }}.service"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
become: true
|
||||
notify:
|
||||
- Reload systemd daemon
|
||||
- Restart nvidia_exporter
|
||||
|
||||
- name: Ensure NVIDIA exporter is enabled and started
|
||||
ansible.builtin.systemd:
|
||||
name: "{{ nvidia_exporter_service_name }}"
|
||||
enabled: true
|
||||
state: started
|
||||
become: true
|
||||
Reference in New Issue
Block a user