feat(ansible): add AI tools setup playbook and related configurations

ansible/ansible.cfg

@@ -2,4 +2,4 @@
 inventory = ../inventory/hosts
 
 [ssh_connection]
-ssh_args = -o ControlMaster=auto -o ControlPersist=60s
+ssh_args = -o ControlMaster=auto -o ControlPersist=60s

ansible/apply.sh

@@ -1,55 +0,0 @@
-#!/bin/bash
-
-# check if sectool is available
-command -v sectool >/dev/null 2>&1 || { echo >&2 "sectool is not installed.  Aborting."; exit 1; }
-
-# check if ansible is available
-command -v ansible >/dev/null 2>&1 || { echo >&2 "ansible is not installed.  Aborting."; exit 1; }
-
-# check if ansible-playbook is available
-command -v ansible-playbook >/dev/null 2>&1 || { echo >&2 "ansible-playbook is not installed.  Aborting."; exit 1; }
-
-function help {
-    echo "Usage: $0 -d <playbook>"
-    echo "Options:"
-    echo "  -d, --dry-run: Run ansible-playbook in dry run mode (don't apply changes)"
-    exit 1
-}
-
-if [ -z $ANSIBLE_USER ]; then
-    echo "ANSIBLE_USER environment variable not set"
-    exit 1
-fi
-
-# parse arguments
-while [ "$1" != "" ]; do
-    case $1 in
-        -d | --dry-run )        shift
-                                DRY_RUN="-DC"
-                                ;;
-        -h | --help )           help
-                                ;;
-        * )                     PLAYBOOK=$1
-                                break
-                                ;;
-    esac
-done
-
-if [ -z "$PLAYBOOK" ]; then
-    echo "Playbook not specified"
-    help
-else
-    if [ ! -f "$PLAYBOOK" ]; then
-        echo "Playbook $PLAYBOOK not found"
-        help
-    fi
-fi
-
-if [ -z "$DRY_RUN" ]; then
-    echo "Applying playbook $PLAYBOOK"
-else
-    echo "Dry run playbook $PLAYBOOK"
-fi
-
-# execute ansible-playbook
-sectool exec ansible-playbook -u $ANSIBLE_USER $DRY_RUN $PLAYBOOK

ansible/files/dockerfiles/Dockerfile.ollama

@@ -0,0 +1,11 @@
+FROM docker.io/ollama/ollama:0.7.1
+
+ARG UID=1000
+ARG GID=1000
+RUN groupadd --system --gid ${GID} worker && \
+    adduser --system --gid ${GID} --uid ${UID} --home /home/worker worker
+
+WORKDIR /home/worker
+USER worker
+ENTRYPOINT ["/bin/ollama"]
+CMD ["serve"]

ansible/files/services/nvidia-cdi-generator.service

@@ -0,0 +1,21 @@
+[Unit]
+Description=NVIDIA: Regenerate NVIDIA CDI configuration at boot
+After=local-fs.target network-online.target
+
+[Service]
+Type=oneshot
+User=root
+Group=root
+
+# This line replicates the logging intent from your script. Output goes to journald.
+ExecStartPre=/bin/sh -c "echo 'nvidia-cdi-generator: regenerating nvidia-container-toolkit configuration' >&2"
+# Remove the existing configuration file (ignore error if it doesn't exist)
+ExecStartPre=-/usr/bin/rm -f /etc/cdi/nvidia.yaml
+# Generate the new configuration file
+ExecStart=/usr/bin/nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
+
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

ansible/host_vars/gpu-01.alexpires.dev.yml

@@ -0,0 +1,100 @@
+hostname: gpu-01.alexpires.local
+ansible_user: "{{ lookup('env', 'ANSIBLE_USER') }}"
+
+# Users
+login_users:
+  - username: "{{ ansible_user }}"
+    comment: "Managed by Ansible"
+    sudoer: true
+    sudoer_root_only: true
+    sudoer_no_password: true
+    pubkey: "{{ lookup('file', 'keys/ansible_user.pub') }}"
+    password_disabled: true
+  - username: operator
+    comment: "Managed by Ansible"
+    sudoer: true
+    pubkey: "{{ lookup('file', 'keys/operator.pub') }}"
+    password_expiration: false
+
+# SSH Connection and security
+sshd_port: 22
+ssh_allowed_networks:
+  - "0.0.0.0/0"
+sshd_admin_net: "0.0.0.0/0"
+sshd_allow_groups: "{{ ansible_user }} users"
+sshd_permit_root_login: "no"
+sshd_password_authentication: "no"
+sshd_allow_tcp_forwarding: "yes"
+
+# Due to an issue increase the number of max auth tries
+sshd_max_auth_tries: 10
+
+packages_blocklist: []
+
+# Firewall ports
+ufw_enable: true
+fw_allowed_ports:
+  - { rule: "allow", port: "22", proto: "tcp" }
+  - { rule: "allow", port: "8080", proto: "tcp" }
+  - { rule: "allow", port: "11434", proto: "tcp" }
+
+ufw_outgoing_traffic:
+  - 22 # SSH
+  - 53 # DNS
+  - 80 # HTTP
+  - 123 # NTP
+  - 443 # HTTPS
+  - 853 # DNS over TLS
+
+ufw_default_forward_policy: "ACCEPT"
+
+ipv4_sysctl_settings:
+  net.ipv4.ip_forward: 1
+
+disable_ipv6: true
+
+# geoip will override hosts_allow and hosts_deny (so it's disabled - CIS hardening)
+custom_hosts_acls: true
+
+# This fixes polkit issues with sudoers file (CIS hardening)
+hide_pid: 0
+
+# aide
+install_aide: false
+
+# Unattended upgrades
+unattended_reboot: true
+unattended_reboot_time: "04:30"
+
+# Duo Security
+duo_users: ["*", "!provision"]
+duo_api_hostname: api-7cda1654.duosecurity.com
+
+# ai tools
+ai_home: "/mnt/data/ai"
+ai_network_name: "ai-network"
+ai_user: "ai"
+ai_group: "ai"
+ai_workloads:
+  - name: ollama
+    build:
+      dockerfile: "{{ lookup('file', 'dockerfiles/Dockerfile.ollama') }}"
+    ports:
+      - 0.0.0.0:11434:11434/tcp
+    volumes:
+      - "{{ ai_home }}/ollama:/home/worker/.ollama:Z"
+    device:
+      - "nvidia.com/gpu=all"
+  - name: open-webui
+    repo: 
+      image: ghcr.io/open-webui/open-webui
+      tag: main
+    ports:
+      - 0.0.0.0:8080:8080/tcp
+    volumes:
+      - "{{ ai_home }}/open-webui:/app/backend/data:Z"
+    device:
+      - "nvidia.com/gpu=all"
+    env:
+      OLLAMA_BASE_URL: "http://ollama:11434"
+

ansible/playbook_ai_tools.yml

@@ -0,0 +1,198 @@
+- name: Setup AI tools with Podman (Ollama)
+  hosts: ai
+
+  tasks:
+    - name: Install Podman and its dependencies
+      become: true
+      ansible.builtin.dnf:
+        name:
+          - podman
+          - nvidia-container-toolkit
+        state: present
+
+    - name: Create group and user for AI workloads
+      become: true
+      ansible.builtin.group:
+        name: "{{ ai_group }}"
+        system: true
+
+    - name: Create user and group for AI containers
+      become: true
+      ansible.builtin.user:
+        name: "{{ ai_user }}"
+        groups: "{{ ai_group }}"
+        shell: /bin/bash
+        home: "{{ ai_home }}"
+        group: "{{ ai_group }}"
+
+    - name: Persistently set SELinux context for home directories on AI home
+      become: true
+      community.general.sefcontext:
+        target: "{{ ai_home }}(/.*)?"
+        setype: user_home_dir_t
+        state: present
+
+    - name: Apply SELinux context changes to AI home directories
+      become: true
+      ansible.builtin.command: restorecon -Rv {{ ai_home }}
+      changed_when: false
+
+    - name: Disable password login for AI user
+      ansible.builtin.command: passwd -d "{{ ai_user }}"
+      become: true
+      changed_when: false
+
+    - name: Create data folders for AI tools
+      become: true
+      ansible.builtin.file:
+        path: "{{ ai_home }}"
+        state: directory
+        owner: "{{ ai_user }}"
+        group: "{{ ai_group }}"
+        mode: "0750"
+
+    - name: Create build folders for AI tools
+      become: true
+      ansible.builtin.file:
+        path: "{{ ai_home }}/build"
+        state: directory
+        owner: "{{ ai_user }}"
+        group: "{{ ai_group }}"
+        mode: "0750"
+
+    - name: Create data folders for AI tools
+      become: true
+      ansible.builtin.file:
+        path: "{{ ai_home }}/{{ item.name }}"
+        state: directory
+        owner: "{{ ai_user }}"
+        group: "{{ ai_group }}"
+        mode: "0750"
+      loop: "{{ ai_workloads }}"
+
+    - name: Create systemd service file for nvidia-cdi-generator
+      become: true
+      ansible.builtin.copy:
+        src: services/nvidia-cdi-generator.service
+        dest: /etc/systemd/system/nvidia-cdi-generator.service
+        owner: root
+        group: root
+        mode: "0644"
+
+    - name: Reload systemd daemon
+      become: true
+      ansible.builtin.systemd:
+        daemon_reload: true
+        name: nvidia-cdi-generator
+
+    - name: Enable nvidia-cdi-generator service
+      become: true
+      ansible.builtin.systemd:
+        name: nvidia-cdi-generator
+        enabled: true
+        state: started
+
+    - name: Enable sudo access to ai user (temporary)
+      become: true
+      ansible.builtin.lineinfile:
+        path: /etc/sudoers.d/ansible_ai_tmp
+        create: true
+        mode: '0440'
+        line: "{{ ansible_user_id }} ALL=(ai) NOPASSWD: ALL"
+        validate: 'visudo -cf %s'
+
+    - name: Enable lingering for ai user
+      become: true
+      ansible.builtin.command: loginctl enable-linger {{ ai_user }}
+      register: linger_status
+      changed_when: "'created' in linger_status.stdout or (linger_status.rc == 0 and not ('linger file already exists' in linger_status.stderr or 'linger file does not exist' in linger_status.stderr))" # Heuristic for change
+      failed_when: linger_status.rc != 0 and not ('linger file already exists' in linger_status.stderr)
+
+    - name: Get current user's UID
+      ansible.builtin.command: id -u {{ ai_user }}
+      changed_when: false
+      register: uid
+
+    - name: Get current user's GID
+      ansible.builtin.command: id -g {{ ai_user }}
+      changed_when: false
+      register: gid
+
+    - name: SHowing UID and GID of ai user
+      ansible.builtin.debug:
+        msg: "UID: {{ uid.stdout }}, GID: {{ gid.stdout }}"
+
+    - name: Run tasks as ai user
+      become: true
+      become_user: "{{ ai_user }}"
+      block:
+        - name: Create a custom Podman network
+          containers.podman.podman_network:
+            name: "{{ ai_network_name }}"
+            state: present
+
+        - name: Build containers images
+          containers.podman.podman_image:
+            name: "ai/{{ item.name }}"
+            path: "{{ ai_home }}/build"
+            build:
+              cache: false
+              force_rm: true
+              container_file: "{{ item.build.dockerfile }}"
+              extra_args: "--build-arg UID={{ uid.stdout }} --build-arg GID={{ gid.stdout }} --security-opt=label=disable {{ item.build.args | default('') }}"
+            tag: "latest"
+            state: present
+          loop: "{{ ai_workloads }}"
+          when: item.build is defined
+
+        - name: Pull the requested images
+          containers.podman.podman_image:
+            name: "{{ item.repo.image }}"
+            tag: "{{ item.repo.tag | default('latest') }}"
+            state: present
+          loop: "{{ ai_workloads }}"
+          when: item.repo is defined
+
+        - name: Create a container instance for each workload
+          containers.podman.podman_container:
+            name: "{{ item.name }}"
+            image: "{{ item.repo.image | default('ai/' + item.name) }}"
+            state: started
+            device: "{{ item.device | default(omit) }}"
+            env: "{{ item.env | default(omit) }}"
+            network: "{{ ai_network_name }}"
+            ports: "{{ item.ports | default(omit) }}"
+            volumes: "{{ item.volumes | default(omit) }}"
+            cmd_args:
+              - "--userns=keep-id"
+              - "--security-opt=label=disable"
+          loop: "{{ ai_workloads }}"
+
+    - name: Create systemd service file for containers
+      become: true
+      ansible.builtin.template:
+        src: podman.service.j2
+        dest: "/etc/systemd/system/podman-{{ item.name }}.service"
+        owner: root
+        group: root
+        mode: "0644"
+      loop: "{{ ai_workloads }}"
+
+    - name: Reload systemd daemon
+      become: true
+      ansible.builtin.systemd:
+        daemon_reload: true
+
+    - name: Enable containers at boot
+      become: true
+      ansible.builtin.systemd:
+        name: "podman-{{ item.name }}"
+        enabled: true
+        state: started
+      loop: "{{ ai_workloads }}"
+
+    - name: Remove temporary sudo access
+      become: true
+      ansible.builtin.file:
+        path: /etc/sudoers.d/ansible_ai_tmp
+        state: absent

ansible/playbook_duo_security.yml

@@ -1,6 +1,6 @@
 ---
 - name: Duo Security
-  hosts: 2fa
+  hosts: duo
   gather_facts: true
 
   roles:

ansible/roles/login/tasks/main.yml

@@ -13,6 +13,7 @@
     shell: "{{ item.shell if item.shell is defined else '/bin/bash' }}"
     groups: "{{ item.groups | default(omit) }}"
     uid: "{{ item.uid | default(omit) }}"
+    home: "/home/{{ item.username }}"
   loop: "{{ login_users }}"
 
 

ansible/templates/podman.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Podman file for {{ item.name }}
+After=local-fs.target network-online.target nvidia-cdi-generator.service
+
+[Service]
+Type=oneshot
+User={{ ai_user}}
+Group={{ ai_group }}
+RemainAfterExit=true
+ExecStart=/usr/sbin/podman start {{ item.name }}
+ExecStop=/usr/sbin/podman stop {{ item.name }}
+
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

inventory/hosts

@@ -1,22 +1,30 @@
 [all]
 prod-01.alexpires.me
 dev-01.alexpires.dev
+vh-01.alexpires.dev
+gpu-01.alexpires.dev
 
 [hardening]
 prod-01.alexpires.me
 dev-01.alexpires.dev
 vh-01.alexpires.dev
+gpu-01.alexpires.dev
 
 [ubuntu]
 prod-01.alexpires.me
 dev-01.alexpires.dev
 
+[fedora]
+vh-01.alexpires.dev
+gpu-01.alexpires.dev
+
 [public]
 prod-01.alexpires.me
 
 [private]
 dev-01.alexpires.dev
 vh-01.alexpires.dev
+gpu-01.alexpires.dev
 
 [gitea]
 prod-01.alexpires.me
@@ -54,7 +62,11 @@ prod-01.alexpires.me
 [virtualization]
 vh-01.alexpires.dev
 
-[2fa]
+[duo]
 prod-01.alexpires.me
 dev-01.alexpires.dev
 vh-01.alexpires.dev
+gpu-01.alexpires.dev
+
+[ai]
+gpu-01.alexpires.dev