From 8dd48ec2c7e90bcdd3934f8d7875caa30ec6a5e1 Mon Sep 17 00:00:00 2001 From: Alexandre Pires Date: Sat, 31 May 2025 17:59:01 +0200 Subject: [PATCH] feat(ansible): add AI tools setup playbook and related configurations --- ansible/ansible.cfg | 2 +- ansible/apply.sh | 55 ----- ansible/files/dockerfiles/Dockerfile.ollama | 11 + .../services/nvidia-cdi-generator.service | 21 ++ ansible/host_vars/gpu-01.alexpires.dev.yml | 100 +++++++++ ansible/playbook_ai_tools.yml | 198 ++++++++++++++++++ ansible/playbook_duo_security.yml | 2 +- ansible/roles/login/tasks/main.yml | 1 + ansible/templates/podman.service.j2 | 17 ++ inventory/hosts | 14 +- 10 files changed, 363 insertions(+), 58 deletions(-) delete mode 100755 ansible/apply.sh create mode 100644 ansible/files/dockerfiles/Dockerfile.ollama create mode 100644 ansible/files/services/nvidia-cdi-generator.service create mode 100644 ansible/host_vars/gpu-01.alexpires.dev.yml create mode 100644 ansible/playbook_ai_tools.yml create mode 100644 ansible/templates/podman.service.j2 diff --git a/ansible/ansible.cfg b/ansible/ansible.cfg index 6eef302..5bd5cd3 100644 --- a/ansible/ansible.cfg +++ b/ansible/ansible.cfg @@ -2,4 +2,4 @@ inventory = ../inventory/hosts [ssh_connection] -ssh_args = -o ControlMaster=auto -o ControlPersist=60s \ No newline at end of file +ssh_args = -o ControlMaster=auto -o ControlPersist=60s diff --git a/ansible/apply.sh b/ansible/apply.sh deleted file mode 100755 index 3c41787..0000000 --- a/ansible/apply.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# check if sectool is available -command -v sectool >/dev/null 2>&1 || { echo >&2 "sectool is not installed. Aborting."; exit 1; } - -# check if ansible is available -command -v ansible >/dev/null 2>&1 || { echo >&2 "ansible is not installed. Aborting."; exit 1; } - -# check if ansible-playbook is available -command -v ansible-playbook >/dev/null 2>&1 || { echo >&2 "ansible-playbook is not installed. Aborting."; exit 1; } - -function help { - echo "Usage: $0 -d " - echo "Options:" - echo " -d, --dry-run: Run ansible-playbook in dry run mode (don't apply changes)" - exit 1 -} - -if [ -z $ANSIBLE_USER ]; then - echo "ANSIBLE_USER environment variable not set" - exit 1 -fi - -# parse arguments -while [ "$1" != "" ]; do - case $1 in - -d | --dry-run ) shift - DRY_RUN="-DC" - ;; - -h | --help ) help - ;; - * ) PLAYBOOK=$1 - break - ;; - esac -done - -if [ -z "$PLAYBOOK" ]; then - echo "Playbook not specified" - help -else - if [ ! -f "$PLAYBOOK" ]; then - echo "Playbook $PLAYBOOK not found" - help - fi -fi - -if [ -z "$DRY_RUN" ]; then - echo "Applying playbook $PLAYBOOK" -else - echo "Dry run playbook $PLAYBOOK" -fi - -# execute ansible-playbook -sectool exec ansible-playbook -u $ANSIBLE_USER $DRY_RUN $PLAYBOOK diff --git a/ansible/files/dockerfiles/Dockerfile.ollama b/ansible/files/dockerfiles/Dockerfile.ollama new file mode 100644 index 0000000..14f9e53 --- /dev/null +++ b/ansible/files/dockerfiles/Dockerfile.ollama @@ -0,0 +1,11 @@ +FROM docker.io/ollama/ollama:0.7.1 + +ARG UID=1000 +ARG GID=1000 +RUN groupadd --system --gid ${GID} worker && \ + adduser --system --gid ${GID} --uid ${UID} --home /home/worker worker + +WORKDIR /home/worker +USER worker +ENTRYPOINT ["/bin/ollama"] +CMD ["serve"] \ No newline at end of file diff --git a/ansible/files/services/nvidia-cdi-generator.service b/ansible/files/services/nvidia-cdi-generator.service new file mode 100644 index 0000000..a583f80 --- /dev/null +++ b/ansible/files/services/nvidia-cdi-generator.service @@ -0,0 +1,21 @@ +[Unit] +Description=NVIDIA: Regenerate NVIDIA CDI configuration at boot +After=local-fs.target network-online.target + +[Service] +Type=oneshot +User=root +Group=root + +# This line replicates the logging intent from your script. Output goes to journald. +ExecStartPre=/bin/sh -c "echo 'nvidia-cdi-generator: regenerating nvidia-container-toolkit configuration' >&2" +# Remove the existing configuration file (ignore error if it doesn't exist) +ExecStartPre=-/usr/bin/rm -f /etc/cdi/nvidia.yaml +# Generate the new configuration file +ExecStart=/usr/bin/nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml + +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/ansible/host_vars/gpu-01.alexpires.dev.yml b/ansible/host_vars/gpu-01.alexpires.dev.yml new file mode 100644 index 0000000..bb86a6e --- /dev/null +++ b/ansible/host_vars/gpu-01.alexpires.dev.yml @@ -0,0 +1,100 @@ +hostname: gpu-01.alexpires.local +ansible_user: "{{ lookup('env', 'ANSIBLE_USER') }}" + +# Users +login_users: + - username: "{{ ansible_user }}" + comment: "Managed by Ansible" + sudoer: true + sudoer_root_only: true + sudoer_no_password: true + pubkey: "{{ lookup('file', 'keys/ansible_user.pub') }}" + password_disabled: true + - username: operator + comment: "Managed by Ansible" + sudoer: true + pubkey: "{{ lookup('file', 'keys/operator.pub') }}" + password_expiration: false + +# SSH Connection and security +sshd_port: 22 +ssh_allowed_networks: + - "0.0.0.0/0" +sshd_admin_net: "0.0.0.0/0" +sshd_allow_groups: "{{ ansible_user }} users" +sshd_permit_root_login: "no" +sshd_password_authentication: "no" +sshd_allow_tcp_forwarding: "yes" + +# Due to an issue increase the number of max auth tries +sshd_max_auth_tries: 10 + +packages_blocklist: [] + +# Firewall ports +ufw_enable: true +fw_allowed_ports: + - { rule: "allow", port: "22", proto: "tcp" } + - { rule: "allow", port: "8080", proto: "tcp" } + - { rule: "allow", port: "11434", proto: "tcp" } + +ufw_outgoing_traffic: + - 22 # SSH + - 53 # DNS + - 80 # HTTP + - 123 # NTP + - 443 # HTTPS + - 853 # DNS over TLS + +ufw_default_forward_policy: "ACCEPT" + +ipv4_sysctl_settings: + net.ipv4.ip_forward: 1 + +disable_ipv6: true + +# geoip will override hosts_allow and hosts_deny (so it's disabled - CIS hardening) +custom_hosts_acls: true + +# This fixes polkit issues with sudoers file (CIS hardening) +hide_pid: 0 + +# aide +install_aide: false + +# Unattended upgrades +unattended_reboot: true +unattended_reboot_time: "04:30" + +# Duo Security +duo_users: ["*", "!provision"] +duo_api_hostname: api-7cda1654.duosecurity.com + +# ai tools +ai_home: "/mnt/data/ai" +ai_network_name: "ai-network" +ai_user: "ai" +ai_group: "ai" +ai_workloads: + - name: ollama + build: + dockerfile: "{{ lookup('file', 'dockerfiles/Dockerfile.ollama') }}" + ports: + - 0.0.0.0:11434:11434/tcp + volumes: + - "{{ ai_home }}/ollama:/home/worker/.ollama:Z" + device: + - "nvidia.com/gpu=all" + - name: open-webui + repo: + image: ghcr.io/open-webui/open-webui + tag: main + ports: + - 0.0.0.0:8080:8080/tcp + volumes: + - "{{ ai_home }}/open-webui:/app/backend/data:Z" + device: + - "nvidia.com/gpu=all" + env: + OLLAMA_BASE_URL: "http://ollama:11434" + diff --git a/ansible/playbook_ai_tools.yml b/ansible/playbook_ai_tools.yml new file mode 100644 index 0000000..e1a205f --- /dev/null +++ b/ansible/playbook_ai_tools.yml @@ -0,0 +1,198 @@ +- name: Setup AI tools with Podman (Ollama) + hosts: ai + + tasks: + - name: Install Podman and its dependencies + become: true + ansible.builtin.dnf: + name: + - podman + - nvidia-container-toolkit + state: present + + - name: Create group and user for AI workloads + become: true + ansible.builtin.group: + name: "{{ ai_group }}" + system: true + + - name: Create user and group for AI containers + become: true + ansible.builtin.user: + name: "{{ ai_user }}" + groups: "{{ ai_group }}" + shell: /bin/bash + home: "{{ ai_home }}" + group: "{{ ai_group }}" + + - name: Persistently set SELinux context for home directories on AI home + become: true + community.general.sefcontext: + target: "{{ ai_home }}(/.*)?" + setype: user_home_dir_t + state: present + + - name: Apply SELinux context changes to AI home directories + become: true + ansible.builtin.command: restorecon -Rv {{ ai_home }} + changed_when: false + + - name: Disable password login for AI user + ansible.builtin.command: passwd -d "{{ ai_user }}" + become: true + changed_when: false + + - name: Create data folders for AI tools + become: true + ansible.builtin.file: + path: "{{ ai_home }}" + state: directory + owner: "{{ ai_user }}" + group: "{{ ai_group }}" + mode: "0750" + + - name: Create build folders for AI tools + become: true + ansible.builtin.file: + path: "{{ ai_home }}/build" + state: directory + owner: "{{ ai_user }}" + group: "{{ ai_group }}" + mode: "0750" + + - name: Create data folders for AI tools + become: true + ansible.builtin.file: + path: "{{ ai_home }}/{{ item.name }}" + state: directory + owner: "{{ ai_user }}" + group: "{{ ai_group }}" + mode: "0750" + loop: "{{ ai_workloads }}" + + - name: Create systemd service file for nvidia-cdi-generator + become: true + ansible.builtin.copy: + src: services/nvidia-cdi-generator.service + dest: /etc/systemd/system/nvidia-cdi-generator.service + owner: root + group: root + mode: "0644" + + - name: Reload systemd daemon + become: true + ansible.builtin.systemd: + daemon_reload: true + name: nvidia-cdi-generator + + - name: Enable nvidia-cdi-generator service + become: true + ansible.builtin.systemd: + name: nvidia-cdi-generator + enabled: true + state: started + + - name: Enable sudo access to ai user (temporary) + become: true + ansible.builtin.lineinfile: + path: /etc/sudoers.d/ansible_ai_tmp + create: true + mode: '0440' + line: "{{ ansible_user_id }} ALL=(ai) NOPASSWD: ALL" + validate: 'visudo -cf %s' + + - name: Enable lingering for ai user + become: true + ansible.builtin.command: loginctl enable-linger {{ ai_user }} + register: linger_status + changed_when: "'created' in linger_status.stdout or (linger_status.rc == 0 and not ('linger file already exists' in linger_status.stderr or 'linger file does not exist' in linger_status.stderr))" # Heuristic for change + failed_when: linger_status.rc != 0 and not ('linger file already exists' in linger_status.stderr) + + - name: Get current user's UID + ansible.builtin.command: id -u {{ ai_user }} + changed_when: false + register: uid + + - name: Get current user's GID + ansible.builtin.command: id -g {{ ai_user }} + changed_when: false + register: gid + + - name: SHowing UID and GID of ai user + ansible.builtin.debug: + msg: "UID: {{ uid.stdout }}, GID: {{ gid.stdout }}" + + - name: Run tasks as ai user + become: true + become_user: "{{ ai_user }}" + block: + - name: Create a custom Podman network + containers.podman.podman_network: + name: "{{ ai_network_name }}" + state: present + + - name: Build containers images + containers.podman.podman_image: + name: "ai/{{ item.name }}" + path: "{{ ai_home }}/build" + build: + cache: false + force_rm: true + container_file: "{{ item.build.dockerfile }}" + extra_args: "--build-arg UID={{ uid.stdout }} --build-arg GID={{ gid.stdout }} --security-opt=label=disable {{ item.build.args | default('') }}" + tag: "latest" + state: present + loop: "{{ ai_workloads }}" + when: item.build is defined + + - name: Pull the requested images + containers.podman.podman_image: + name: "{{ item.repo.image }}" + tag: "{{ item.repo.tag | default('latest') }}" + state: present + loop: "{{ ai_workloads }}" + when: item.repo is defined + + - name: Create a container instance for each workload + containers.podman.podman_container: + name: "{{ item.name }}" + image: "{{ item.repo.image | default('ai/' + item.name) }}" + state: started + device: "{{ item.device | default(omit) }}" + env: "{{ item.env | default(omit) }}" + network: "{{ ai_network_name }}" + ports: "{{ item.ports | default(omit) }}" + volumes: "{{ item.volumes | default(omit) }}" + cmd_args: + - "--userns=keep-id" + - "--security-opt=label=disable" + loop: "{{ ai_workloads }}" + + - name: Create systemd service file for containers + become: true + ansible.builtin.template: + src: podman.service.j2 + dest: "/etc/systemd/system/podman-{{ item.name }}.service" + owner: root + group: root + mode: "0644" + loop: "{{ ai_workloads }}" + + - name: Reload systemd daemon + become: true + ansible.builtin.systemd: + daemon_reload: true + + - name: Enable containers at boot + become: true + ansible.builtin.systemd: + name: "podman-{{ item.name }}" + enabled: true + state: started + loop: "{{ ai_workloads }}" + + - name: Remove temporary sudo access + become: true + ansible.builtin.file: + path: /etc/sudoers.d/ansible_ai_tmp + state: absent diff --git a/ansible/playbook_duo_security.yml b/ansible/playbook_duo_security.yml index d41e887..906a0dc 100644 --- a/ansible/playbook_duo_security.yml +++ b/ansible/playbook_duo_security.yml @@ -1,6 +1,6 @@ --- - name: Duo Security - hosts: 2fa + hosts: duo gather_facts: true roles: diff --git a/ansible/roles/login/tasks/main.yml b/ansible/roles/login/tasks/main.yml index 6608cf4..001e7af 100644 --- a/ansible/roles/login/tasks/main.yml +++ b/ansible/roles/login/tasks/main.yml @@ -13,6 +13,7 @@ shell: "{{ item.shell if item.shell is defined else '/bin/bash' }}" groups: "{{ item.groups | default(omit) }}" uid: "{{ item.uid | default(omit) }}" + home: "/home/{{ item.username }}" loop: "{{ login_users }}" diff --git a/ansible/templates/podman.service.j2 b/ansible/templates/podman.service.j2 new file mode 100644 index 0000000..583aa2f --- /dev/null +++ b/ansible/templates/podman.service.j2 @@ -0,0 +1,17 @@ +[Unit] +Description=Podman file for {{ item.name }} +After=local-fs.target network-online.target nvidia-cdi-generator.service + +[Service] +Type=oneshot +User={{ ai_user}} +Group={{ ai_group }} +RemainAfterExit=true +ExecStart=/usr/sbin/podman start {{ item.name }} +ExecStop=/usr/sbin/podman stop {{ item.name }} + +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/inventory/hosts b/inventory/hosts index 4301a88..6363f78 100644 --- a/inventory/hosts +++ b/inventory/hosts @@ -1,22 +1,30 @@ [all] prod-01.alexpires.me dev-01.alexpires.dev +vh-01.alexpires.dev +gpu-01.alexpires.dev [hardening] prod-01.alexpires.me dev-01.alexpires.dev vh-01.alexpires.dev +gpu-01.alexpires.dev [ubuntu] prod-01.alexpires.me dev-01.alexpires.dev +[fedora] +vh-01.alexpires.dev +gpu-01.alexpires.dev + [public] prod-01.alexpires.me [private] dev-01.alexpires.dev vh-01.alexpires.dev +gpu-01.alexpires.dev [gitea] prod-01.alexpires.me @@ -54,7 +62,11 @@ prod-01.alexpires.me [virtualization] vh-01.alexpires.dev -[2fa] +[duo] prod-01.alexpires.me dev-01.alexpires.dev vh-01.alexpires.dev +gpu-01.alexpires.dev + +[ai] +gpu-01.alexpires.dev