feat: Update configuration for Nextcloud and Certbot, add image and video organization scripts

ansible/host_vars/gpu-01.lab.alexpires.me.yml

@@ -47,6 +47,7 @@ fw_allowed_ports:
   - { rule: "allow", port: "8999", proto: "tcp", from: "10.19.4.0/24" } # ComfyUI - SimpleHttpServer (local network)
   - { rule: "allow", port: "8012", proto: "tcp", from: "10.19.4.0/24" } # Qwen2 (local network)
   - { rule: "allow", port: "8012", proto: "tcp", from: "10.5.5.5/32" } # Qwen2 (Laptop)
+  - { rule: "allow", port: "9090", proto: "tcp", from: "10.19.4.0/24" } # local network
 ufw_outgoing_traffic:
   - 22 # SSH
   - 53 # DNS
@@ -95,3 +96,13 @@ comfyui_user_pubkey: "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbm
 
 # Kokoro FastAPI settings (we don't set home because it is not expected to use a lot a disk space)
 kokoro_user_pubkey: "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAHHOPBR9p9kq5Cqzpe4cr3jHnweaYrHPXv5sXNzt+sCXP54uc5rWUBhxW2OQVvQzJ47dEVhEKi4WSC7LcuKS2G5AQDzWXNiasHvYIYQU3F/EknVCZnsiXYqXphYkJA6rJCNRnISZCIC1mocq6PB7J08ONdRFCvjfUBuVRT8BNGXNmQ/zQ=="
+
+# Let's Encrypt / Certbot
+cloudns_auth_id: "{{ lookup('env', 'CLOUDNS_AUTH_ID') }}"
+cloudns_auth_password: "{{ lookup('env', 'CLOUDNS_PASSWORD') }}"
+cloudns_nameserver: "109.201.133.111"
+certbot_email: c.alexandre.pires@alexpires.me
+certbot_domains:
+  - "gpu-01.lab.alexpires.me"
+
+certbot_cockpit_domain: "gpu-01.lab.alexpires.me"

ansible/host_vars/vh-01.lab.alexpires.me.yml

@@ -19,6 +19,23 @@ packages_blocklist:
   - yp-tools
   - ypbind
 
+misc_modules_blocklist:
+  - bluetooth
+  - bnep
+  - btusb
+  - can
+  - cpia2
+  - firewire-core
+  - floppy
+  - ksmbd
+  - n_hdlc
+  - net-pf-31
+  - pcspkr
+  - soundcore
+  - usb-midi
+  - uvcvideo
+  - v4l2_common
+
 # Users
 login_users:
   - username: "{{ ansible_user }}"
@@ -106,7 +123,7 @@ virt_vfio_devices:
   - { dev: "0000:05:00.0", vendor: "c0a9", device: "5415" } # Micron/Crucial Technology T500 NVMe PCIe SSD
 
 virt_extra_grub_args: >
-  rd.driver.blacklist=nouveau modprobe.blacklist=nouveau ipv6.disable=1 audit=1 audit_backlog_limit=8192
+  rd.driver.blacklist=nouveau modprobe.blacklist=nouveau ipv6.disable=1 audit=1 audit_backlog_limit=8192 usbcore.autosuspend=-1
 
 virt_vfio_ids:
   - 10de:1430 # NVIDIA Corporation M2000 [Maxwell GM107GL]
@@ -142,6 +159,9 @@ virt_machines:
     mac_address: "52:54:00:cd:dd:2d"
     block_devs: []
     guest_agent: true
+    virtio_fs:
+      - source: "/mnt/usb"
+        target: "usb"
     host_devs:
       # Micron/Crucial Technology T500 NVMe PCIe SSD
       - type: "pci"
@@ -221,6 +241,9 @@ virt_machines:
     mac_address: "52:54:00:cd:dd:2e"
     block_devs: []
     guest_agent: true
+    virtio_fs:
+      - source: "/mnt/usb"
+        target: "usb"
     host_devs:
       # Micron/Crucial Technology T500 NVMe PCIe SSD
       - type: "pci"
@@ -306,8 +329,8 @@ virt_machines:
     virtio_fs:
       - source: "/mnt/srv"
         target: "srv"
-      - source: "/mnt/storage"
+      - source: "/mnt/md0"
-        target: "storage"
+        target: "md0"
     user_data: |
       #cloud-config
       hostname: dev-01
@@ -336,3 +359,4 @@ cloudns_nameserver: "109.201.133.111"
 certbot_email: c.alexandre.pires@alexpires.me
 certbot_domains:
   - "vh-01.lab.alexpires.me"
+certbot_cockpit_domain: "vh-01.lab.alexpires.me"

ansible/playbook_certbot.yml

@@ -18,19 +18,35 @@
       ansible.builtin.fail:
         msg: "CloudDNS nameserver is not set"
 
-    - name: Install Certbot and dependencies
+    - name: Remove Certbot installed via package manager (Debian/Ubuntu)
+      when: ansible_os_family == "Debian"
+      ansible.builtin.apt:
+        name:
+          - certbot
+        state: absent
+
+    - name: Remove Certbot installed via package manager (RedHat/CentOS)
       when: ansible_os_family == "RedHat"
       ansible.builtin.dnf:
         name:
           - certbot
-        state: present
+        state: absent
 
     - name: Install cloudDNS plugin dependencies
       ansible.builtin.pip:
         name:
+          - certbot
           - certbot-dns-cloudns
         state: present
 
+    - name: Ensure CloudDNS credentials directory exists
+      ansible.builtin.file:
+        path: "/etc/letsencrypt"
+        state: directory
+        owner: root
+        group: root
+        mode: '0700'
+
     - name: Create ClouDNS credentials file
       ansible.builtin.copy:
         dest: "{{ cloudns_credentials_path }}"
@@ -59,3 +75,18 @@
         job: "certbot renew --quiet"
         minute: 0
         hour: 3
+
+    - name: Setup cockpit certifcate (if defined)
+      when: certbot_cockpit_domain is defined
+      block:
+        - name: Create symlink for Cockpit certificate
+          ansible.builtin.file:
+            src: "/etc/letsencrypt/live/{{ certbot_cockpit_domain }}/fullchain.pem"
+            dest: "/etc/cockpit/ws-certs.d/99-letsencrypt-cert.cert"
+            state: link
+
+        - name: Create symlink for Cockpit private key
+          ansible.builtin.file:
+            src: "/etc/letsencrypt/live/{{ certbot_cockpit_domain }}/privkey.pem"
+            dest: "/etc/cockpit/ws-certs.d/99-letsencrypt-cert.key"
+            state: link

inventory/hosts

@@ -18,6 +18,10 @@ dev-01.lab.alexpires.me
 vh-01.lab.alexpires.me
 gpu-01.lab.alexpires.me
 
+[certbot]
+vh-01.lab.alexpires.me
+gpu-01.lab.alexpires.me
+
 [public]
 prod-01.alexpires.me
 

scripts/organize_images.sh

@@ -0,0 +1,182 @@
+#!/bin/bash
+
+# Parse arguments
+DRY_RUN=false
+INPUT_DIR=""
+OUTPUT_DIR=""
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --dry-run|-n)
+            DRY_RUN=true
+            shift
+            ;;
+        --input|-i)
+            INPUT_DIR="$2"
+            shift 2
+            ;;
+        --output|-o)
+            OUTPUT_DIR="$2"
+            shift 2
+            ;;
+        --help|-h)
+            echo "Usage: $0 --input DIR --output DIR [OPTIONS]"
+            echo ""
+            echo "Required:"
+            echo "  --input, -i DIR      Input directory containing images"
+            echo "  --output, -o DIR     Output directory for organized images"
+            echo ""
+            echo "Options:"
+            echo "  --dry-run, -n        Preview changes without moving files"
+            echo "  --help, -h           Show this help message"
+            exit 0
+            ;;
+        *)
+            echo "Unknown option: $1"
+            echo "Use --help for usage information"
+            exit 1
+            ;;
+    esac
+done
+
+# Validate required arguments
+if [[ -z "$INPUT_DIR" ]]; then
+    echo "Error: --input directory is required"
+    echo "Use --help for usage information"
+    exit 1
+fi
+
+if [[ -z "$OUTPUT_DIR" ]]; then
+    echo "Error: --output directory is required"
+    echo "Use --help for usage information"
+    exit 1
+fi
+
+# Check if input directory exists
+if [[ ! -d "$INPUT_DIR" ]]; then
+    echo "Error: Input directory '$INPUT_DIR' does not exist"
+    exit 1
+fi
+
+# Check if exiftool is installed
+if ! command -v exiftool &> /dev/null; then
+    echo "Error: exiftool is not installed. Please install it first."
+    echo "  Ubuntu/Debian: sudo apt-get install libimage-exiftool-perl"
+    echo "  macOS: brew install exiftool"
+    exit 1
+fi
+
+# Create output directory (only if not dry-run)
+if [[ "$DRY_RUN" == false ]]; then
+    mkdir -p "$OUTPUT_DIR"
+fi
+
+# Counters
+processed=0
+unknown=0
+failed=0
+
+if [[ "$DRY_RUN" == true ]]; then
+    echo "🔍 DRY RUN MODE - No files will be moved"
+fi
+echo "Starting image organization..."
+echo "Input directory: $INPUT_DIR"
+echo "Output directory: $OUTPUT_DIR"
+echo ""
+
+# Process each image
+# Using -iname for case-insensitive matching (matches .jpg, .JPG, .Jpg, etc.)
+while IFS= read -r image; do
+    echo "Processing: $(basename "$image")"
+    
+    # Extract creation date (YYYYMMDD)
+    # Try DateTimeOriginal first (most common for photos), then CreateDate, then ModifyDate
+    creation_date=$(exiftool -DateTimeOriginal -d "%Y%m%d" -s3 "$image" 2>/dev/null | head -n 1 | tr -d ' ')
+    
+    if [[ -z "$creation_date" ]] || [[ ! "$creation_date" =~ ^[0-9]{8}$ ]]; then
+        creation_date=$(exiftool -CreateDate -d "%Y%m%d" -s3 "$image" 2>/dev/null | head -n 1 | tr -d ' ')
+    fi
+    
+    if [[ -z "$creation_date" ]] || [[ ! "$creation_date" =~ ^[0-9]{8}$ ]]; then
+        creation_date=$(exiftool -ModifyDate -d "%Y%m%d" -s3 "$image" 2>/dev/null | head -n 1 | tr -d ' ')
+    fi
+    
+    # Check if date was extracted successfully
+    if [[ -z "$creation_date" ]] || [[ ! "$creation_date" =~ ^[0-9]{8}$ ]]; then
+        echo "  ⚠ Warning: Could not extract valid date, moving to Unknown folder"
+        target_dir="$OUTPUT_DIR/Unknown"
+        destination="$target_dir/$(basename "$image")"
+        
+        # Check if destination file already exists
+        if [[ -f "$destination" ]]; then
+            # Add timestamp to avoid overwrite
+            filename=$(basename "$image")
+            extension="${filename##*.}"
+            basename_no_ext="${filename%.*}"
+            destination="$target_dir/${basename_no_ext}_$(date +%s).$extension"
+            echo "  ⚠ File exists, renaming to: $(basename "$destination")"
+        fi
+        
+        if [[ "$DRY_RUN" == true ]]; then
+            echo "  → Would move to: Unknown/$(basename "$destination")"
+        else
+            mkdir -p "$target_dir"
+            if mv "$image" "$destination" 2>/dev/null; then
+                echo "  ✓ Moved to: Unknown/"
+            else
+                echo "  ✗ Failed to move file"
+                failed=$((failed + 1))
+                continue
+            fi
+        fi
+        unknown=$((unknown + 1))
+        continue
+    fi
+    
+    # Extract year, month, day
+    year="${creation_date:0:4}"
+    month="${creation_date:4:2}"
+    day="${creation_date:6:2}"
+    
+    # Create subdirectories (YYYY/MM/DD)
+    target_dir="$OUTPUT_DIR/$year/$month/$day"
+    
+    # Check if destination file already exists
+    destination="$target_dir/$(basename "$image")"
+    if [[ -f "$destination" ]]; then
+        # Add timestamp to avoid overwrite
+        filename=$(basename "$image")
+        extension="${filename##*.}"
+        basename_no_ext="${filename%.*}"
+        destination="$target_dir/${basename_no_ext}_$(date +%s).$extension"
+        echo "  ⚠ File exists, renaming to: $(basename "$destination")"
+    fi
+    
+    if [[ "$DRY_RUN" == true ]]; then
+        # Dry run: just show what would happen
+        echo "  → Would move to: $year/$month/$day/$(basename "$destination")"
+        processed=$((processed + 1))
+    else
+        # Actually perform the move
+        mkdir -p "$target_dir"
+        if mv "$image" "$destination" 2>/dev/null; then
+            echo "  ✓ Moved to: $year/$month/$day/"
+            processed=$((processed + 1))
+        else
+            echo "  ✗ Failed to move file"
+            failed=$((failed + 1))
+        fi
+    fi
+done < <(find "$INPUT_DIR" -type f \( -iname "*.jpg" -o -iname "*.jpeg" -o -iname "*.png" -o -iname "*.gif" -o -iname "*.bmp" -o -iname "*.tiff" -o -iname "*.tif" -o -iname "*.heic" -o -iname "*.heif" -o -iname "*.raw" -o -iname "*.cr2" -o -iname "*.nef" -o -iname "*.arw" -o -iname "*.dng" -o -iname "*.webp" \))
+
+echo ""
+if [[ "$DRY_RUN" == true ]]; then
+    echo "DRY RUN Summary:"
+    echo "  Would process: $processed"
+    echo "  Would move to Unknown: $unknown"
+else
+    echo "Summary:"
+    echo "  Processed: $processed"
+    echo "  Moved to Unknown: $unknown"
+    echo "  Failed: $failed"
+fi

scripts/organize_videos.sh

@@ -0,0 +1,173 @@
+#!/bin/bash
+
+# Parse arguments
+DRY_RUN=false
+INPUT_DIR=""
+OUTPUT_DIR=""
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --dry-run|-n)
+            DRY_RUN=true
+            shift
+            ;;
+        --input|-i)
+            INPUT_DIR="$2"
+            shift 2
+            ;;
+        --output|-o)
+            OUTPUT_DIR="$2"
+            shift 2
+            ;;
+        --help|-h)
+            echo "Usage: $0 --input DIR --output DIR [OPTIONS]"
+            echo ""
+            echo "Required:"
+            echo "  --input, -i DIR      Input directory containing videos"
+            echo "  --output, -o DIR     Output directory for organized videos"
+            echo ""
+            echo "Options:"
+            echo "  --dry-run, -n        Preview changes without moving files"
+            echo "  --help, -h           Show this help message"
+            exit 0
+            ;;
+        *)
+            echo "Unknown option: $1"
+            echo "Use --help for usage information"
+            exit 1
+            ;;
+    esac
+done
+
+# Validate required arguments
+if [[ -z "$INPUT_DIR" ]]; then
+    echo "Error: --input directory is required"
+    echo "Use --help for usage information"
+    exit 1
+fi
+
+if [[ -z "$OUTPUT_DIR" ]]; then
+    echo "Error: --output directory is required"
+    echo "Use --help for usage information"
+    exit 1
+fi
+
+# Check if input directory exists
+if [[ ! -d "$INPUT_DIR" ]]; then
+    echo "Error: Input directory '$INPUT_DIR' does not exist"
+    exit 1
+fi
+
+# Check if exiftool is installed
+if ! command -v exiftool &> /dev/null; then
+    echo "Error: exiftool is not installed. Please install it first."
+    echo "  Ubuntu/Debian: sudo apt-get install libimage-exiftool-perl"
+    echo "  macOS: brew install exiftool"
+    exit 1
+fi
+
+# Create output directory (only if not dry-run)
+if [[ "$DRY_RUN" == false ]]; then
+    mkdir -p "$OUTPUT_DIR"
+fi
+
+# Counters
+processed=0
+skipped=0
+failed=0
+
+if [[ "$DRY_RUN" == true ]]; then
+    echo "🔍 DRY RUN MODE - No files will be moved"
+fi
+echo "Starting video organization..."
+echo "Input directory: $INPUT_DIR"
+echo "Output directory: $OUTPUT_DIR"
+echo ""
+
+# Process each video
+# Using -iname for case-insensitive matching (matches .mp4, .MP4, .Mp4, etc.)
+while IFS= read -r video; do
+    echo "Processing: $(basename "$video")"
+    
+    # Extract creation date (YYYYMMDD)
+    creation_date=$(exiftool -MediaCreateDate -d "%Y%m%d" -s3 "$video" 2>/dev/null | head -n 1 | tr -d ' ')
+    
+    # Check if date was extracted successfully
+    if [[ -z "$creation_date" ]] || [[ ! "$creation_date" =~ ^[0-9]{8}$ ]]; then
+        echo "  ⚠ Warning: Could not extract valid date, moving to Unknown folder"
+        target_dir="$OUTPUT_DIR/Unknown"
+        destination="$target_dir/$(basename "$video")"
+        
+        # Check if destination file already exists
+        if [[ -f "$destination" ]]; then
+            # Add timestamp to avoid overwrite
+            filename=$(basename "$video")
+            extension="${filename##*.}"
+            basename_no_ext="${filename%.*}"
+            destination="$target_dir/${basename_no_ext}_$(date +%s).$extension"
+            echo "  ⚠ File exists, renaming to: $(basename "$destination")"
+        fi
+        
+        if [[ "$DRY_RUN" == true ]]; then
+            echo "  → Would move to: Unknown/$(basename "$destination")"
+        else
+            mkdir -p "$target_dir"
+            if mv "$video" "$destination" 2>/dev/null; then
+                echo "  ✓ Moved to: Unknown/"
+            else
+                echo "  ✗ Failed to move file"
+                failed=$((failed + 1))
+                continue
+            fi
+        fi
+        skipped=$((skipped + 1))
+        continue
+    fi
+    
+    # Extract year, month, day
+    year="${creation_date:0:4}"
+    month="${creation_date:4:2}"
+    day="${creation_date:6:2}"
+    
+    # Create subdirectories (YYYY/MM/DD)
+    target_dir="$OUTPUT_DIR/$year/$month/$day"
+    
+    # Check if destination file already exists
+    destination="$target_dir/$(basename "$video")"
+    if [[ -f "$destination" ]]; then
+        # Add timestamp to avoid overwrite
+        filename=$(basename "$video")
+        extension="${filename##*.}"
+        basename_no_ext="${filename%.*}"
+        destination="$target_dir/${basename_no_ext}_$(date +%s).$extension"
+        echo "  ⚠ File exists, renaming to: $(basename "$destination")"
+    fi
+    
+    if [[ "$DRY_RUN" == true ]]; then
+        # Dry run: just show what would happen
+        echo "  → Would move to: $year/$month/$day/$(basename "$destination")"
+        processed=$((processed + 1))
+    else
+        # Actually perform the move
+        mkdir -p "$target_dir"
+        if mv "$video" "$destination" 2>/dev/null; then
+            echo "  ✓ Moved to: $year/$month/$day/"
+            processed=$((processed + 1))
+        else
+            echo "  ✗ Failed to move file"
+            failed=$((failed + 1))
+        fi
+    fi
+done < <(find "$INPUT_DIR" -type f \( -iname "*.mp4" -o -iname "*.mov" -o -iname "*.mkv" -o -iname "*.avi" -o -iname "*.wmv" -o -iname "*.flv" -o -iname "*.webm" \))
+
+echo ""
+if [[ "$DRY_RUN" == true ]]; then
+    echo "DRY RUN Summary:"
+    echo "  Would process: $processed"
+    echo "  Would move to Unknown: $skipped"
+else
+    echo "Summary:"
+    echo "  Processed: $processed"
+    echo "  Moved to Unknown: $skipped"
+    echo "  Failed: $failed"
+fi

terraform/apps/dev-01/apps.tf

@@ -35,7 +35,7 @@ module "nextcloud" {
   source = "../../modules/apps/nextcloud"
 
   fqdn                       = local.nextcloud_fqdn
-  persistent_folder          = local.persistent_folder
+  nextcloud_data             = local.nextcloud_data
   app_config                 = local.nextcloud_config
   db_credentials_are_secrets = true
 

terraform/apps/dev-01/config.tf

@@ -1,7 +1,7 @@
 locals {
 
   # storage
-  persistent_folder = "/srv"
+  persistent_folder = "/mnt/srv"
 
   lab_domain      = "lab.alexpires.me"
   internal_issuer = "internal-ca"
@@ -14,7 +14,7 @@ locals {
   samba_shares = [
     {
       name    = "Storage"
-      path    = "/mnt/storage"
+      path    = "/mnt/md0/storage"
       comment = "Main Storage"
     }
   ]
@@ -40,7 +40,7 @@ locals {
         {
           name    = "vh-01"
           type    = "A"
-          content = "10.19.4.109"
+          content = "10.19.4.254"
         },
         {
           name    = "gpu-01"
@@ -176,8 +176,9 @@ locals {
   nextcloud_fqdn             = "cloud.alexpires.me"
   nextcloud_db_passwd        = var.nextcloud_db_passwd
   nextcloud_db_root_password = var.nextcloud_db_root_password
-  nextcloud_db_path          = "${local.persistent_folder}/mysql/nextcloud"
+  nextcloud_db_path          = "/mnt/md0/nextcloud.db"
-  backup_folder              = "/mnt/usb/backups/nextcloud"
+  nextcloud_backup           = "/mnt/md0/nextcloud.backups"
+  nextcloud_data             = "/mnt/md0/nextcloud.files"
   nextcloud_config = {
     mail_from      = "cloud"
     mail_domain    = "alexpires.me"

terraform/apps/dev-01/databases.tf

@@ -10,7 +10,7 @@ module "nextcloud_db" {
   cron_schedule  = local.backup_cron_schedule
 
   database_folder = local.nextcloud_db_path
-  backup_folder   = local.backup_folder
+  backup_folder   = local.nextcloud_backup
 
   tls_enabled = true
   issuer      = local.internal_issuer

terraform/modules/apps/nextcloud/config.tf

@@ -1,6 +1,4 @@
 locals {
-  persistent_folder = var.persistent_folder
-
   database_credentials = {
     MYSQL_USER = {
       key  = "username"
@@ -63,9 +61,6 @@ locals {
     }
   }, var.db_credentials_are_secrets ? local.database_credentials : {})
 
-
-  nextcloud_data = "${local.persistent_folder}/nextcloud"
-
   apache_confs = {
     for f in fileset("${path.module}/resources/apache2/conf-enabled/", "*.conf") :
     "${f}" => file("${path.module}/resources/apache2/conf-enabled//${f}")

terraform/modules/apps/nextcloud/main.tf

@@ -229,7 +229,7 @@ resource "kubernetes_deployment" "nextcloud_apache" {
         volume {
           name = "html-data"
           host_path {
-            path = local.nextcloud_data
+            path = var.nextcloud_data
           }
         }
         volume {

terraform/modules/apps/nextcloud/variables.tf

@@ -1,4 +1,4 @@
-variable "persistent_folder" {
+variable "nextcloud_data" {
   description = "The path to the persistent folder"
   type        = string
 }