feat(comfyui): add Dockerfile, playbook, and Terraform configurations for ComfyUI setup

This commit is contained in:
2025-06-08 21:18:53 +02:00
parent d969525b58
commit 359b6d47aa
18 changed files with 400 additions and 170 deletions
@@ -0,0 +1,35 @@
# Use NVIDIA's PyTorch image with CUDA support
FROM docker.io/pytorch/pytorch:2.1.0-cuda11.8-cudnn8-runtime
# Set environment variables
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
# Install system dependencies
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
apt-get install -y tzdata git ffmpeg libgl1 libglib2.0-0 && \
rm -rf /var/lib/apt/lists/*
ARG UID=1000
ARG GID=1000
RUN groupadd --system --gid ${GID} worker && \
adduser --system --gid ${GID} --uid ${UID} --home /home/worker worker
WORKDIR /home/worker
USER worker
# Clone ComfyUI
RUN git clone https://github.com/comfyanonymous/ComfyUI.git .
# Install Python dependencies
RUN pip install --upgrade pip && \
pip install -r requirements.txt && \
rm -rf /home/worker/.cache/pip
# Expose port
EXPOSE 8188
# Run ComfyUI
CMD ["python", "main.py", "--listen", "0.0.0.0", "--port", "8188"]
+11 -3
View File
@@ -40,7 +40,8 @@ fw_allowed_ports:
- { rule: "allow", port: "11434", proto: "tcp", from: "10.19.4.0/24" } # Ollama (local network) - { rule: "allow", port: "11434", proto: "tcp", from: "10.19.4.0/24" } # Ollama (local network)
- { rule: "allow", port: "11434", proto: "tcp", from: "10.5.5.5/32" } # Ollama (Laptop) - { rule: "allow", port: "11434", proto: "tcp", from: "10.5.5.5/32" } # Ollama (Laptop)
- { rule: "allow", port: "8880", proto: "tcp", from: "10.19.4.0/24" } # Kokoro-FastAPI (local network) - { rule: "allow", port: "8880", proto: "tcp", from: "10.19.4.0/24" } # Kokoro-FastAPI (local network)
- { rule: "allow", port: "8188", proto: "tcp", from: "10.19.4.0/24" } # ComfyUI (local network)
- { rule: "allow", port: "8999", proto: "tcp", from: "10.19.4.0/24" } # ComfyUI - SimpleHttpServer (local network)
ufw_outgoing_traffic: ufw_outgoing_traffic:
- 22 # SSH - 22 # SSH
- 53 # DNS - 53 # DNS
@@ -70,9 +71,16 @@ unattended_reboot: true
unattended_reboot_time: "04:30" unattended_reboot_time: "04:30"
# Duo Security # Duo Security
duo_users: ["*", "!provision"] duo_users: ["*", "!provision", "!comfyui", "!kokoro", "!ollama"]
duo_api_hostname: api-7cda1654.duosecurity.com duo_api_hostname: api-7cda1654.duosecurity.com
# podman services settings # podman services settings
ollama_home: "/mnt/data/ollama" ollama_home: "/mnt/data/ollama"
kokora_home: "/mnt/data/kokoro" podman_user_pubkey: "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAHHOPBR9p9kq5Cqzpe4cr3jHnweaYrHPXv5sXNzt+sCXP54uc5rWUBhxW2OQVvQzJ47dEVhEKi4WSC7LcuKS2G5AQDzWXNiasHvYIYQU3F/EknVCZnsiXYqXphYkJA6rJCNRnISZCIC1mocq6PB7J08ONdRFCvjfUBuVRT8BNGXNmQ/zQ=="
# ComfyUI settings
comfyui_home: "/mnt/data/comfyui"
comfyui_user_pubkey: "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAHHOPBR9p9kq5Cqzpe4cr3jHnweaYrHPXv5sXNzt+sCXP54uc5rWUBhxW2OQVvQzJ47dEVhEKi4WSC7LcuKS2G5AQDzWXNiasHvYIYQU3F/EknVCZnsiXYqXphYkJA6rJCNRnISZCIC1mocq6PB7J08ONdRFCvjfUBuVRT8BNGXNmQ/zQ=="
# Kokoro FastAPI settings (we don't set home because it is not expected to use a lot a disk space)
kokoro_user_pubkey: "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAHHOPBR9p9kq5Cqzpe4cr3jHnweaYrHPXv5sXNzt+sCXP54uc5rWUBhxW2OQVvQzJ47dEVhEKi4WSC7LcuKS2G5AQDzWXNiasHvYIYQU3F/EknVCZnsiXYqXphYkJA6rJCNRnISZCIC1mocq6PB7J08ONdRFCvjfUBuVRT8BNGXNmQ/zQ=="
+58
View File
@@ -0,0 +1,58 @@
- name: Setup ComfyUI
hosts: comfyui
vars:
podman_user: "{{ comfyui_user | default('comfyui') }}"
podman_group: "{{ comfyui_group | default('comfyui') }}"
podman_user_home: "{{ comfyui_home | default('/home/comfyui') }}"
podman_extra_groups: "users"
podman_user_pubkey: "{{ comfyui_user_pubkey | default(omit) }}"
podman_user_folders:
- models
- models/checkpoints
- models/clip
- models/clip_vision
- models/configs
- models/controlnet
- models/diffusers
- models/diffusion_models
- models/embeddings
- models/gligen
- models/hypernetworks
- models/loras
- models/photomaker
- models/style_models
- models/text_encoders
- models/unet
- models/upscale_models
- models/vae
- models/vae_approx
- output
- custom_nodes
- .cache
pods:
- name: comfyui
build:
dockerfile: "{{ lookup('file', 'dockerfiles/Dockerfile.comfyui') }}"
ports:
- "0.0.0.0:{{ comfyui_port | default(8188) }}:8188/tcp"
env:
PYTORCH_CUDA_ALLOC_CONF: max_split_size_mb:512
volumes:
- "{{ podman_user_home }}/.cache:/home/worker/.cache:Z"
- "{{ podman_user_home }}/models:/home/worker/models:Z"
- "{{ podman_user_home }}/custom_nodes:/home/worker/custom_nodes:Z"
- "{{ podman_user_home }}/output:/home/worker/output:Z"
device:
- "nvidia.com/gpu=all"
systemd:
enabled: false
state: "stopped"
tasks:
- name: Setup nvidia-cdi-generator
ansible.builtin.include_tasks:
file: tasks/nvidia-cdi-generator.yml
- name: Setup Pods
ansible.builtin.include_tasks:
file: tasks/podman.yml
+6
View File
@@ -3,7 +3,9 @@
vars: vars:
podman_user: "{{ kokoro_user | default('kokoro') }}" podman_user: "{{ kokoro_user | default('kokoro') }}"
podman_group: "{{ kokoro_group | default('kokoro') }}" podman_group: "{{ kokoro_group | default('kokoro') }}"
podman_extra_groups: "users"
podman_user_home: "{{ kokoro_home | default('/home/kokoro') }}" podman_user_home: "{{ kokoro_home | default('/home/kokoro') }}"
podman_user_pubkey: "{{ kokoro_user_pubkey | default(omit) }}"
podman_user_folders: podman_user_folders:
- data - data
pods: pods:
@@ -16,6 +18,10 @@
- "nvidia.com/gpu=all" - "nvidia.com/gpu=all"
tasks: tasks:
- name: Setup nvidia-cdi-generator
ansible.builtin.include_tasks:
file: tasks/nvidia-cdi-generator.yml
- name: Setup Pods - name: Setup Pods
ansible.builtin.include_tasks: ansible.builtin.include_tasks:
file: tasks/podman.yml file: tasks/podman.yml
-33
View File
@@ -1,33 +0,0 @@
- name: Setup MCP servers
hosts: mcp
vars:
podman_user: "{{ mcp_user | default('mcp') }}"
podman_group: "{{ mcp_group | default('mcp') }}"
podman_user_home: "{{ mcp_home | default('/home/mcp') }}"
mcp_servers:
- name: gitea-mcp
enabled: false
build:
super_user_custom_actions: |
RUN apt-get update && apt-get install -y --no-install-recommends \
curl \
tar \
&& rm -rf /var/lib/apt/lists/*
RUN curl -L https://gitea.com/gitea/gitea-mcp/releases/download/v0.2.0/gitea-mcp_Linux_x86_64.tar.gz \
| tar -xz -C /usr/local/bin \
&& chmod +x /usr/local/bin/gitea-mcp
worker_custom_actions: |
ENV GITEA_HOST="https://gitea.com"
ENV GITEA_INSECURE="false"
ENV GITEA_ACCESS_TOKEN="<your personal access token>"
mcp_server_commands: ["gitea-mcp"]
port: 8000
env:
GITEA_HOST: "https://code.alexpires.me"
GITEA_ACCESS_TOKEN: "{{ lookup('env', 'GITEA_APP_TOKEN') }}"
tasks:
- name: Setup Pods
ansible.builtin.include_tasks:
file: tasks/mcp-proxy.yml
loop: "{{ mcp_servers }}"
+1
View File
@@ -3,6 +3,7 @@
vars: vars:
podman_user: "{{ ollama_user | default('ollama') }}" podman_user: "{{ ollama_user | default('ollama') }}"
podman_group: "{{ ollama_group | default('ollama') }}" podman_group: "{{ ollama_group | default('ollama') }}"
podman_extra_groups: "users"
podman_user_home: "{{ ollama_home | default('/home/ollama') }}" podman_user_home: "{{ ollama_home | default('/home/ollama') }}"
podman_user_folders: podman_user_folders:
- data - data
+18 -113
View File
@@ -24,67 +24,9 @@
- acl - acl
state: present state: present
- name: Create group and user for workloads - name: Setup user
become: true ansible.builtin.include_tasks:
ansible.builtin.group: file: podman/user.yml
name: "{{ podman_group }}"
system: true
- name: Create user and group for containers
become: true
ansible.builtin.user:
name: "{{ podman_user }}"
groups: "{{ podman_group }}"
shell: /bin/bash
home: "{{ podman_user_home }}"
group: "{{ podman_group }}"
- name: Disable password login for user
ansible.builtin.command: passwd -d "{{ podman_user }}"
become: true
changed_when: false
- name: SELinux tasks (RedHat only)
when: ansible_os_family == 'RedHat'
become: true
block:
- name: Persistently set SELinux context for home directories on AI home
community.general.sefcontext:
target: "{{ podman_user_home }}(/.*)?"
setype: user_home_dir_t
state: present
- name: Apply SELinux context changes to AI home directories
ansible.builtin.command: restorecon -Rv {{ podman_user_home }}
changed_when: false
- name: Create user home
become: true
ansible.builtin.file:
path: "{{ podman_user_home }}"
state: directory
owner: "{{ podman_user }}"
group: "{{ podman_group }}"
mode: "0750"
- name: Create build folders
become: true
ansible.builtin.file:
path: "{{ podman_user_home }}/build"
state: directory
owner: "{{ podman_user }}"
group: "{{ podman_group }}"
mode: "0750"
- name: Create folders
become: true
ansible.builtin.file:
path: "{{ podman_user_home }}/{{ item }}"
state: directory
owner: "{{ podman_user }}"
group: "{{ podman_group }}"
mode: "0750"
loop: "{{ podman_user_folders }}"
- name: Enable sudo access to podman user (temporary) - name: Enable sudo access to podman user (temporary)
become: true become: true
@@ -118,53 +60,6 @@
changed_when: false changed_when: false
register: gid register: gid
- name: Ensure rootless Podman works with overlay driver on Debian-family systems
when: ansible_os_family == 'Debian'
become: true
vars:
subuid_range: "100000:65536"
block:
- name: Install required packages for rootless Podman overlay support
ansible.builtin.apt:
name:
- uidmap
- fuse-overlayfs
- libslirp0
- slirp4netns
state: present
update_cache: true
- name: Ensure subuid entry exists for the user
ansible.builtin.lineinfile:
path: /etc/subuid
line: "{{ podman_user }}:{{ subuid_range }}"
create: true
state: present
regexp: "^{{ podman_user }}:"
mode: "0644"
- name: Ensure subgid entry exists for the user
ansible.builtin.lineinfile:
path: /etc/subgid
line: "{{ podman_user }}:{{ subuid_range }}"
create: true
state: present
regexp: "^{{ podman_user }}:"
mode: "0644"
- name: Ensure overlay module is loaded # noqa no-ignore-errors
community.general.modprobe:
name: overlay
state: present
- name: Ensure overlay module loads on boot
ansible.builtin.lineinfile:
path: /etc/modules
line: overlay
create: true
state: present
mode: "0644"
- name: Run tasks as podman user - name: Run tasks as podman user
become: true become: true
become_user: "{{ podman_user }}" become_user: "{{ podman_user }}"
@@ -175,6 +70,16 @@
name: "{{ podman_network_name }}" name: "{{ podman_network_name }}"
state: present state: present
- name: Copy build files to user home
ansible.builtin.copy:
src: "{{ item }}"
dest: "{{ podman_user_home }}/build/"
owner: "{{ podman_user }}"
group: "{{ podman_group }}"
mode: "0644"
loop: "{{ podman_build_files | default([]) }}"
when: podman_build_files is defined and podman_build_files | length > 0
- name: Build containers images - name: Build containers images
containers.podman.podman_image: containers.podman.podman_image:
name: "{{ podman_user }}/{{ item.name }}" name: "{{ podman_user }}/{{ item.name }}"
@@ -187,7 +92,7 @@
tag: "latest" tag: "latest"
state: present state: present
loop: "{{ pods }}" loop: "{{ pods }}"
when: item.build is defined when: item.build is defined and item.build.dockerfile is defined
- name: Pull the requested images - name: Pull the requested images
containers.podman.podman_image: containers.podman.podman_image:
@@ -195,13 +100,13 @@
tag: "{{ item.repo.tag | default('latest') }}" tag: "{{ item.repo.tag | default('latest') }}"
state: present state: present
loop: "{{ pods }}" loop: "{{ pods }}"
when: item.repo is defined when: item.repo is defined and item.repo.image is defined
- name: Create pods - name: Create pods
containers.podman.podman_container: containers.podman.podman_container:
name: "{{ item.name }}" name: "{{ item.name }}"
image: "{{ item.repo.image | default(podman_user + '/' + item.name) }}" image: "{{ item.repo.image | default(podman_user + '/' + item.name) }}"
state: started state: "{{ item.state | default('present') }}"
device: "{{ item.device | default(omit) }}" device: "{{ item.device | default(omit) }}"
env: "{{ item.env | default(omit) }}" env: "{{ item.env | default(omit) }}"
network: "{{ podman_network_name | default(omit) }}" network: "{{ podman_network_name | default(omit) }}"
@@ -231,8 +136,8 @@
become: true become: true
ansible.builtin.systemd: ansible.builtin.systemd:
name: "podman-{{ item.name }}" name: "podman-{{ item.name }}"
enabled: true enabled: "{{ item.systemd.enabled | default(true) }}"
state: started state: "{{ item.systemd.state | default('started') }}"
loop: "{{ pods }}" loop: "{{ pods }}"
- name: Remove temporary sudo access - name: Remove temporary sudo access
+84
View File
@@ -0,0 +1,84 @@
- name: Check if required vars are set
ansible.builtin.fail:
msg: "Required variables are not set for Podman setup. Please check your playbook variables.)"
when:
- podman_user is not defined
- podman_group is not defined
- podman_user_home is not defined
- name: Create group
become: true
ansible.builtin.group:
name: "{{ podman_group }}"
system: true
- name: Create user
become: true
ansible.builtin.user:
name: "{{ podman_user }}"
groups: "{{ podman_extra_groups | default([]) }}"
shell: /bin/bash
home: "{{ podman_user_home }}"
group: "{{ podman_group }}"
- name: Disable password login for user
ansible.builtin.command: passwd -d "{{ podman_user }}"
become: true
changed_when: false
- name: Create user home
become: true
ansible.builtin.file:
path: "{{ podman_user_home }}"
state: directory
owner: "{{ podman_user }}"
group: "{{ podman_group }}"
mode: "0750"
- name: Create build folders
become: true
ansible.builtin.file:
path: "{{ podman_user_home }}/build"
state: directory
owner: "{{ podman_user }}"
group: "{{ podman_group }}"
mode: "0750"
- name: Create folders
become: true
ansible.builtin.file:
path: "{{ podman_user_home }}/{{ item }}"
state: directory
owner: "{{ podman_user }}"
group: "{{ podman_group }}"
mode: "0750"
loop: "{{ podman_user_folders }}"
- name: "Add SSH Authorized key"
become: true
ansible.posix.authorized_key:
user: "{{ podman_user }}"
key: "{{ podman_user_pubkey }}"
state: "{{ 'present' if podman_user_pubkey is defined else 'absent' }}"
when: podman_user_pubkey is defined and podman_user_pubkey != ""
- name: SELinux tasks (RedHat only)
when: ansible_os_family == 'RedHat'
become: true
block:
- name: Persistently set SELinux context
community.general.sefcontext:
target: "{{ podman_user_home }}(/.*)?"
setype: user_home_dir_t
state: present
- name: Persistently set SELinux context (ssh authorized keys)
community.general.sefcontext:
target: "{{ podman_user_home }}/.ssh/authorized_keys"
setype: ssh_home_t
state: present
when: podman_user_pubkey is defined and podman_user_pubkey != ""
- name: Apply SELinux context
ansible.builtin.command: restorecon -Rv {{ podman_user_home }}
changed_when: false
+3
View File
@@ -75,6 +75,9 @@ gpu-01.lab.alexpires.me
[kokoro] [kokoro]
gpu-01.lab.alexpires.me gpu-01.lab.alexpires.me
[comfyui]
gpu-01.lab.alexpires.me
[openwebui] [openwebui]
dev-01.lab.alexpires.me dev-01.lab.alexpires.me
+6
View File
@@ -17,3 +17,9 @@ module "dns" {
forward_dns_servers = local.dns_nameservers forward_dns_servers = local.dns_nameservers
zones = local.zones zones = local.zones
} }
module "reverse_proxy" {
source = "../../modules/apps/nginx-reverse-proxy"
services = local.reverse_proxy_services
}
+30
View File
@@ -50,6 +50,16 @@ locals {
type = "CNAME" type = "CNAME"
content = "dev-01.${local.lab_domain}." content = "dev-01.${local.lab_domain}."
}, },
{
name = "comfyui-backend"
type = "CNAME"
content = "gpu-01.${local.lab_domain}."
},
{
name = "comfyui"
type = "CNAME"
content = "dev-01.${local.lab_domain}."
},
{ {
name = "smb" name = "smb"
type = "CNAME" type = "CNAME"
@@ -58,4 +68,24 @@ locals {
] ]
} }
] ]
reverse_proxy_services = {
"comfyui" = {
http_port = 8080
https_port = 8443
tls = true
fqdn = ["comfyui.lab.alexpires.me"]
upstream = "http://comfyui-backend.lab.alexpires.me:8188"
resolver = "10.19.4.136"
locations = [
{
path = "/"
"Upgrade" = "$http_upgrade"
"Connection" = "upgrade"
},
]
}
}
} }
+4
View File
@@ -0,0 +1,4 @@
locals {
comfyui_version = "frontend${var.tag}"
}
+98
View File
@@ -0,0 +1,98 @@
resource "kubernetes_namespace" "comfyui" {
metadata {
name = "comfyui"
}
}
resource "kubernetes_service_account" "comfyui" {
metadata {
name = "comfyui-sa"
namespace = kubernetes_namespace.comfyui.metadata[0].name
}
automount_service_account_token = false
}
resource "kubernetes_deployment" "comfyui" {
metadata {
name = "comfyui"
namespace = kubernetes_namespace.comfyui.metadata[0].name
labels = {
comfyui = "comfyui"
}
}
spec {
replicas = 1
selector {
match_labels = {
comfyui = "comfyui"
}
}
strategy {
type = "RollingUpdate"
}
template {
metadata {
labels = {
comfyui = "comfyui"
}
}
spec {
service_account_name = kubernetes_service_account.comfyui.metadata[0].name
automount_service_account_token = false
container {
name = "comfyui"
image = "a13labs/comfyui:${local.comfyui_version}"
security_context {
allow_privilege_escalation = false
}
port {
name = "frontend"
container_port = 5173
protocol = "TCP"
}
env {
name = "DEV_SERVER_COMFYUI_URL"
value = coalesce(var.backend, "http://localhost:8188")
}
}
}
}
}
}
resource "kubernetes_service" "comfyui" {
metadata {
name = "frontend"
namespace = kubernetes_namespace.comfyui.metadata[0].name
}
spec {
selector = {
comfyui = "comfyui"
}
port {
name = "frontend"
port = 5173
target_port = 5173
}
}
lifecycle {
ignore_changes = [
metadata[0].annotations
]
}
}
@@ -0,0 +1,9 @@
terraform {
required_version = "~>1.8"
required_providers {
kubernetes = {
source = "hashicorp/kubernetes"
version = "2.36.0"
}
}
}
@@ -0,0 +1,11 @@
variable "tag" {
description = "The version of app to install"
type = string
default = "v0.0.1"
}
variable "backend" {
description = "The backend to use for ComfyUI"
type = string
default = ""
}
@@ -170,6 +170,7 @@ resource "kubernetes_ingress_v1" "reverse-proxy_ingress" {
} }
resource "kubernetes_secret" "auth_token" { resource "kubernetes_secret" "auth_token" {
count = var.auth_token != "" ? 1 : 0
metadata { metadata {
name = "auth-token-secret" name = "auth-token-secret"
namespace = kubernetes_namespace.reverse_proxy.metadata[0].name namespace = kubernetes_namespace.reverse_proxy.metadata[0].name
@@ -281,7 +282,9 @@ resource "kubernetes_deployment" "reverse_proxy" {
} }
} }
container { dynamic "container" {
for_each = var.auth_token != "" ? [1] : []
content {
name = "auth-sidecar" name = "auth-sidecar"
image = "python:3.11-slim" image = "python:3.11-slim"
@@ -291,7 +294,7 @@ resource "kubernetes_deployment" "reverse_proxy" {
name = "AUTH_TOKEN" name = "AUTH_TOKEN"
value_from { value_from {
secret_key_ref { secret_key_ref {
name = "auth-token-secret" name = kubernetes_secret.auth_token[0].metadata[0].name
key = "token" key = "token"
} }
} }
@@ -308,6 +311,7 @@ resource "kubernetes_deployment" "reverse_proxy" {
name = "auth" name = "auth"
} }
} }
}
container { container {
name = "reverse-proxy" name = "reverse-proxy"
@@ -39,10 +39,10 @@ location = /auth {
%{ if l.private ~} %{ if l.private ~}
auth_request /auth<; auth_request /auth<;
%{ endif ~} %{ endif ~}
set $upstream "%{ if l.upstream != "" ~}${l.upstream}%{ else ~}${upstream}%{ endif ~}";
%{ if l.rewrite != "" ~} %{ if l.rewrite != "" ~}
rewrite ${l.rewrite}; rewrite ${l.rewrite};
%{ endif ~} %{ endif ~}
set $upstream "%{ if l.upstream != "" ~}${l.upstream}%{ else ~}${upstream}%{ endif ~}";
proxy_pass $upstream; proxy_pass $upstream;
proxy_http_version 1.1; proxy_http_version 1.1;
%{ for k,v in l.headers ~} %{ for k,v in l.headers ~}
@@ -30,7 +30,7 @@ variable "services" {
description = "A list of services to be deployed" description = "A list of services to be deployed"
type = map(object({ type = map(object({
fqdn = list(string) fqdn = list(string)
upstream = string upstream = optional(string, "")
resolver = optional(string, "") resolver = optional(string, "")
default_headers = optional(map(string), {}), default_headers = optional(map(string), {}),
http_port = optional(number, 80) http_port = optional(number, 80)
@@ -53,4 +53,5 @@ variable "auth_token" {
description = "The authentication token to use for the reverse proxy" description = "The authentication token to use for the reverse proxy"
type = string type = string
sensitive = true sensitive = true
default = ""
} }