a13labs.infra/ansible/playbook_virt_host.yml

- name: Setup GPU passthrough host (Fedora + QEMU/KVM)
  hosts: virtualization
  become: true
  vars:
    grub_config_file: "/etc/default/grub"
    grub_cmdline_args: >
      rhgb quiet intel_iommu=on iommu=pt
      isolcpus={{ virt_vm_reserved_cpu }} nohz_full={{ virt_vm_reserved_cpu }} rcu_nocbs={{ virt_vm_reserved_cpu }}
      snd_hda_codec_hdmi.blacklist=1 rd.driver.pre=vfio-pci audit=off
      default_hugepagesz=1G hugepagesz=1G hugepages=8 mitigations=off loglevel=3 {{ virt_extra_grub_args | default("") | trim}}

  tasks:
    - name: Ensure the system is Fedora
      ansible.builtin.assert:
        that:
          - ansible_facts['distribution'] == "Fedora"
        fail_msg: "This playbook only supports Fedora hosts."

    - name: Install python3-libdnf5 so Ansible DNF module works
      become: true
      ansible.builtin.shell:
        cmd: |
          dnf install -y python3-libdnf5
          touch /etc/.python3_libdnf5_installed
      args:
        creates: /etc/.python3_libdnf5_installed

    - name: Install required packages
      ansible.builtin.dnf:
        name:
          - qemu-kvm
          - libvirt
          - virt-install
          - virt-manager
          - bridge-utils
          - pciutils
          - tuned
          - python3-lxml
          - python3-libvirt
          - tuned
          - cloud-utils
          - qemu-img
        state: present

    # ===== BRIDGE CONFIGURATION TASKS =====

    - name: Detect current network interface
      ansible.builtin.set_fact:
        primary_interface: "{{ bridge_interface }}"
      when: bridge_interface is defined

    - name: Check if bridge already exists
      ansible.builtin.command: nmcli -t -f NAME,TYPE connection show
      register: nmcli_connections
      changed_when: false
      failed_when: false

    - name: Set bridge existence fact
      ansible.builtin.set_fact:
        bridge_exists: "{{ (bridge_name in nmcli_connections.stdout) and ('bridge' in nmcli_connections.stdout) }}"

    - name: Get current interface connection name
      ansible.builtin.shell:
        cmd: |
          set -o pipefail
          nmcli -t -f NAME,DEVICE connection show --active | grep ":{{ primary_interface }}$" | cut -d: -f1 | head -n1
        executable: /bin/bash
      register: current_connection_name
      changed_when: false
      when: not bridge_exists

    - name: Create bridge connection with static IPv4
      ansible.builtin.command: >
        nmcli connection add
        type bridge
        con-name {{ bridge_name }}
        ifname {{ bridge_name }}
        ipv4.method manual
        ipv4.addresses {{ bridge_ipv4_address }}
        ipv4.gateway {{ bridge_ipv4_gateway }}
        ipv4.dns "{{ bridge_ipv4_dns }}"
        ipv4.dns-search {{ bridge_ipv4_domain }}
        ipv6.method disabled
        connection.autoconnect yes
        bridge.stp no
      when: not bridge_exists
      changed_when: true
      register: bridge_created

    - name: Create bridge slave connection for physical interface
      ansible.builtin.command: >
        nmcli connection add
        type ethernet
        con-name {{ bridge_name }}-slave
        ifname {{ primary_interface }}
        master {{ bridge_name }}
        slave-type bridge
      when: not bridge_exists and bridge_created is succeeded
      changed_when: true
      register: slave_created

    - name: Deactivate old interface connection and activate bridge
      become: true
      ansible.builtin.command: >
        nmcli connection down "{{ current_connection_name.stdout }}"
        nmcli connection up {{ bridge_name }}
      when:
        - not bridge_exists
        - slave_created is succeeded
        - current_connection_name.stdout | length > 0
      register: bridge_activated
      changed_when: true
      failed_when: false
      async: 10
      poll: 0

    - name: Wait for network connectivity to restore # noqa: no-handler
      ansible.builtin.wait_for:
        host: "{{ bridge_ipv4_address.split('/')[0] }}"
        port: 22
        timeout: 30
      delegate_to: localhost
      when: bridge_activated is changed

    - name: Switch ansible to connect via bridge IP # noqa: no-handler
      ansible.builtin.set_fact:
        ansible_host: "{{ bridge_ipv4_address.split('/')[0] }}"
      when: bridge_activated is changed

    - name: Verify bridge is active # noqa: no-handler
      ansible.builtin.command: ip addr show {{ bridge_name }}
      register: bridge_status
      changed_when: false
      failed_when: "'state UP' not in bridge_status.stdout"
      when: bridge_activated is changed

    - name: Display bridge configuration
      ansible.builtin.debug: # noqa: no-handler
        msg:
          - "Bridge {{ bridge_name }} is configured and active"
          - "IP Address: {{ bridge_ipv4_address }}"
          - "Gateway: {{ bridge_ipv4_gateway }}"
          - "DNS: {{ bridge_ipv4_dns }}"
          - "Physical interface: {{ primary_interface }}"
      when: bridge_activated is changed

    # ===== END BRIDGE CONFIGURATION =====

    - name: Enable and start libvirtd
      ansible.builtin.systemd:
        name: libvirtd
        enabled: true
        state: started

    - name: Ensure GRUB_CMDLINE_LINUX is configured for vfio, isolcpus, nohz_full, rcu_nocbs
      ansible.builtin.lineinfile:
        path: "{{ grub_config_file }}"
        regexp: '^GRUB_CMDLINE_LINUX='
        line: 'GRUB_CMDLINE_LINUX="{{ grub_cmdline_args | trim }}"'
        backrefs: true
      notify: Rebuild GRUB config

    - name: Copy vfio-bind script
      ansible.builtin.copy:
        src: vfio-bind.sh
        dest: /usr/local/bin/vfio-bind.sh
        mode: '0755'

    - name: Create systemd service for vfio-bind
      ansible.builtin.copy:
        dest: "/etc/systemd/system/vfio-{{ item.vendor }}-{{ item.device }}-bind.service"
        content: |
          [Unit]
          Description=Bind PCI device {{ item.vendor }}:{{ item.device }} to VFIO driver
          After=systemd-udev-settle.service
          Wants=systemd-udev-settle.service
          Before=libvirtd.service

          [Service]
          Type=oneshot
          ExecStart=/usr/local/bin/vfio-bind.sh "{{ item.dev }}" "{{ item.vendor }}" "{{ item.device }}"
          RemainAfterExit=yes

          [Install]
          WantedBy=multi-user.target
        mode: '0644'
      loop: "{{ virt_vfio_devices }}"

    - name: Enable and start VFIO bind services
      ansible.builtin.systemd:
        name: "vfio-{{ item.vendor }}-{{ item.device }}-bind.service"
        enabled: true
        state: started
        daemon_reload: true
      loop: "{{ virt_vfio_devices }}"

    - name: Add VFIO PCI device IDs to modprobe
      ansible.builtin.copy:
        dest: /etc/modprobe.d/vfio.conf
        content: |
          options vfio-pci ids={{ virt_vfio_ids | join(',') }}
          install snd_hda_codec_hdmi /bin/false
          softdep xhci_hcd pre: vfio-pci
        mode: '0644'

    - name: Blacklist Nouveau and other drivers
      ansible.builtin.template:
        src: blacklist.conf.j2
        dest: /etc/modprobe.d/ansible-blacklist.conf
        mode: '0644'
      when: virt_blacklist is defined

    - name: Add vfio modules to initramfs config
      ansible.builtin.copy:
        dest: /etc/dracut.conf.d/vfio.conf
        content: |
          add_drivers+=" vfio vfio_iommu_type1 vfio_pci "
        mode: '0644'
      notify: Rebuild initramfs

    - name: Configure hugepages
      ansible.posix.sysctl:
        name: vm.nr_hugepages
        value: "{{ hugepages_count | default('2048') }}"
        state: present
        reload: true

    - name: Enable KVM modules at boot
      ansible.builtin.copy:
        dest: /etc/modules-load.d/kvm.conf
        content: |
          kvmgt
          mdev
          vfio
          vfio-pci
          vfio_iommu_type1
        mode: '0644'

    - name: Set I/O scheduler to 'none' for NVMes
      ansible.builtin.copy:
        dest: /etc/udev/rules.d/60-ioscheduler.rules
        content: |
          ACTION=="add|change", KERNEL=="nvme[0-9]*", ATTR{queue/scheduler}="none"
        mode: '0644'
      notify:
        - Reload udev rules
        - Trigger udev to apply changes

    - name: Activate virtual-host tuned profile
      ansible.builtin.command: tuned-adm profile virtual-host
      changed_when: false

    - name: Set permissions for partition passthrough
      ansible.builtin.file:
        path: "/dev/{{ item }}"
        owner: qemu
        group: kvm
        mode: '0660'
      loop: "{{ virt_raw_block_devices }}"

    - name: Create udev rules for persistent access
      ansible.builtin.template:
        src: 99-qemu-nvme.rules.j2
        dest: /etc/udev/rules.d/99-qemu-nvme.rules
        mode: '0644'
      notify:
        - Reload udev rules
        - Trigger udev to apply changes

    - name: Download and resize cloud image for VM
      block:
        - name: Check if local image already exists
          vars:
            vm: "{{ item }}"
          ansible.builtin.stat:
            path: "/var/lib/libvirt/images/{{ vm.disk_file }}"
            get_checksum: false
          register: cloud_image_stat
          loop: "{{ virt_machines }}"

        - name: Download cloud image if not present
          vars:
            id: "{{ item }}"
          loop: "{{ range(virt_machines | length) }}"
          ansible.builtin.get_url:
            url: "{{ virt_machines[id].cloud_image_url }}"
            dest: "/var/lib/libvirt/images/{{ virt_machines[id].disk_file }}"
            mode: '0644'
            checksum: "sha256:{{ virt_machines[id].cloud_image_sha256 }}"
          when:
            - not cloud_image_stat.results[id].stat.exists
            - virt_machines[id].cloud_image_url is defined
            - virt_machines[id].cloud_image_sha256 is defined

        - name: Resize cloud image
          vars:
            id: "{{ item }}"
          loop: "{{ range(virt_machines | length) }}"
          ansible.builtin.command: >
            qemu-img resize /var/lib/libvirt/images/{{ virt_machines[id].disk_file }} {{ virt_machines[id].disk_size | default('20G') }}
          when:
            - not cloud_image_stat.results[id].stat.exists
            - virt_machines[id].disk_size is defined
          changed_when: false

    - name: Create cloud-init ISO for VM
      block:
        - name: Create temporary cloud-init directory
          vars:
            vm: "{{ item }}"
          loop: "{{ virt_machines }}"
          ansible.builtin.file:
            path: "/var/lib/libvirt/images/{{ vm.name }}-cloudinit"
            state: directory
            mode: '0755'
          when: vm.user_data is defined

        - name: Create user-data file
          vars:
            vm: "{{ item }}"
          loop: "{{ virt_machines }}"
          ansible.builtin.copy:
            dest: "/var/lib/libvirt/images/{{ vm.name }}-cloudinit/user-data"
            content: |
              {{ vm.user_data }}
            mode: '0644'
          when: vm.user_data is defined
          register: user_data_created

        - name: Create meta-data file
          vars:
            vm: "{{ item }}"
          loop: "{{ virt_machines }}"
          ansible.builtin.copy:
            dest: "/var/lib/libvirt/images/{{ vm.name }}-cloudinit/meta-data"
            content: |
              instance-id: {{ vm.name }}-{{ vm.uuid }}
              local-hostname: {{ vm.name }}
            mode: '0644'
          when: vm.user_data is defined

        - name: Create cloud-init ISO image
          vars:
            vm: "{{ item }}"
          loop: "{{ virt_machines }}"
          ansible.builtin.command: >
            cloud-localds
            /var/lib/libvirt/images/{{ vm.name }}-cloudinit.img
            /var/lib/libvirt/images/{{ vm.name }}-cloudinit/user-data
            /var/lib/libvirt/images/{{ vm.name }}-cloudinit/meta-data
          args:
            creates: "/var/lib/libvirt/images/{{ vm.name }}-cloudinit.img"
          when: vm.user_data is defined

    - name: Create or update virtual machine
      vars:
        vm: "{{ item }}"
      loop: "{{ virt_machines }}"
      community.libvirt.virt:
        autostart: "{{ vm.autostart | default(false) }}"
        name: "{{ vm.name }}"
        command: define
        xml: "{{ lookup('template', 'domain.xml.j2') }}"

    - name: Ensure virtual machines are running
      vars:
        vm: "{{ item }}"
      loop: "{{ virt_machines }}"
      community.libvirt.virt:
        name: "{{ vm.name }}"
        state: running
      when: vm.autostart | default(false)

    - name: Notify user to reboot for changes
      ansible.builtin.debug:
        msg: |
          Reboot your system to apply IOMMU, CPU isolation, and VFIO settings.
          After reboot, verify using: `dmesg | grep -i vfio` and `lspci -nnk`

  handlers:
    - name: Reload udev rules
      changed_when: false
      ansible.builtin.command: udevadm control --reload-rules

    - name: Trigger udev to apply changes
      changed_when: false
      ansible.builtin.command: udevadm trigger

    - name: Rebuild initramfs
      changed_when: false
      ansible.builtin.command: dracut -f

    - name: Rebuild GRUB config
      changed_when: false
      ansible.builtin.command: grub2-mkconfig -o /boot/grub2/grub.cfg
      when: ansible_facts['distribution'] == "Fedora"