From 2c8b753393b7771267460c237d4a33be1c624fb3 Mon Sep 17 00:00:00 2001 From: Alexandre Pires Date: Thu, 8 Jan 2026 23:39:55 +0100 Subject: [PATCH] feat: Update configuration for Nextcloud and Certbot, add image and video organization scripts --- ansible/host_vars/gpu-01.lab.alexpires.me.yml | 11 ++ ansible/host_vars/vh-01.lab.alexpires.me.yml | 30 ++- ansible/playbook_certbot.yml | 35 +++- inventory/hosts | 4 + scripts/organize_images.sh | 182 ++++++++++++++++++ scripts/organize_videos.sh | 173 +++++++++++++++++ terraform/apps/dev-01/apps.tf | 2 +- terraform/apps/dev-01/config.tf | 11 +- terraform/apps/dev-01/databases.tf | 2 +- terraform/modules/apps/nextcloud/config.tf | 5 - terraform/modules/apps/nextcloud/main.tf | 2 +- terraform/modules/apps/nextcloud/variables.tf | 2 +- 12 files changed, 440 insertions(+), 19 deletions(-) create mode 100644 scripts/organize_images.sh create mode 100644 scripts/organize_videos.sh diff --git a/ansible/host_vars/gpu-01.lab.alexpires.me.yml b/ansible/host_vars/gpu-01.lab.alexpires.me.yml index d6dbdaf..bbc8d30 100644 --- a/ansible/host_vars/gpu-01.lab.alexpires.me.yml +++ b/ansible/host_vars/gpu-01.lab.alexpires.me.yml @@ -47,6 +47,7 @@ fw_allowed_ports: - { rule: "allow", port: "8999", proto: "tcp", from: "10.19.4.0/24" } # ComfyUI - SimpleHttpServer (local network) - { rule: "allow", port: "8012", proto: "tcp", from: "10.19.4.0/24" } # Qwen2 (local network) - { rule: "allow", port: "8012", proto: "tcp", from: "10.5.5.5/32" } # Qwen2 (Laptop) + - { rule: "allow", port: "9090", proto: "tcp", from: "10.19.4.0/24" } # local network ufw_outgoing_traffic: - 22 # SSH - 53 # DNS @@ -95,3 +96,13 @@ comfyui_user_pubkey: "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbm # Kokoro FastAPI settings (we don't set home because it is not expected to use a lot a disk space) kokoro_user_pubkey: "ecdsa-sha2-nistp521 AAAAE2VjZHNhLXNoYTItbmlzdHA1MjEAAAAIbmlzdHA1MjEAAACFBAHHOPBR9p9kq5Cqzpe4cr3jHnweaYrHPXv5sXNzt+sCXP54uc5rWUBhxW2OQVvQzJ47dEVhEKi4WSC7LcuKS2G5AQDzWXNiasHvYIYQU3F/EknVCZnsiXYqXphYkJA6rJCNRnISZCIC1mocq6PB7J08ONdRFCvjfUBuVRT8BNGXNmQ/zQ==" + +# Let's Encrypt / Certbot +cloudns_auth_id: "{{ lookup('env', 'CLOUDNS_AUTH_ID') }}" +cloudns_auth_password: "{{ lookup('env', 'CLOUDNS_PASSWORD') }}" +cloudns_nameserver: "109.201.133.111" +certbot_email: c.alexandre.pires@alexpires.me +certbot_domains: + - "gpu-01.lab.alexpires.me" + +certbot_cockpit_domain: "gpu-01.lab.alexpires.me" diff --git a/ansible/host_vars/vh-01.lab.alexpires.me.yml b/ansible/host_vars/vh-01.lab.alexpires.me.yml index ec44a94..33c0057 100644 --- a/ansible/host_vars/vh-01.lab.alexpires.me.yml +++ b/ansible/host_vars/vh-01.lab.alexpires.me.yml @@ -19,6 +19,23 @@ packages_blocklist: - yp-tools - ypbind +misc_modules_blocklist: + - bluetooth + - bnep + - btusb + - can + - cpia2 + - firewire-core + - floppy + - ksmbd + - n_hdlc + - net-pf-31 + - pcspkr + - soundcore + - usb-midi + - uvcvideo + - v4l2_common + # Users login_users: - username: "{{ ansible_user }}" @@ -106,7 +123,7 @@ virt_vfio_devices: - { dev: "0000:05:00.0", vendor: "c0a9", device: "5415" } # Micron/Crucial Technology T500 NVMe PCIe SSD virt_extra_grub_args: > - rd.driver.blacklist=nouveau modprobe.blacklist=nouveau ipv6.disable=1 audit=1 audit_backlog_limit=8192 + rd.driver.blacklist=nouveau modprobe.blacklist=nouveau ipv6.disable=1 audit=1 audit_backlog_limit=8192 usbcore.autosuspend=-1 virt_vfio_ids: - 10de:1430 # NVIDIA Corporation M2000 [Maxwell GM107GL] @@ -142,6 +159,9 @@ virt_machines: mac_address: "52:54:00:cd:dd:2d" block_devs: [] guest_agent: true + virtio_fs: + - source: "/mnt/usb" + target: "usb" host_devs: # Micron/Crucial Technology T500 NVMe PCIe SSD - type: "pci" @@ -221,6 +241,9 @@ virt_machines: mac_address: "52:54:00:cd:dd:2e" block_devs: [] guest_agent: true + virtio_fs: + - source: "/mnt/usb" + target: "usb" host_devs: # Micron/Crucial Technology T500 NVMe PCIe SSD - type: "pci" @@ -306,8 +329,8 @@ virt_machines: virtio_fs: - source: "/mnt/srv" target: "srv" - - source: "/mnt/storage" - target: "storage" + - source: "/mnt/md0" + target: "md0" user_data: | #cloud-config hostname: dev-01 @@ -336,3 +359,4 @@ cloudns_nameserver: "109.201.133.111" certbot_email: c.alexandre.pires@alexpires.me certbot_domains: - "vh-01.lab.alexpires.me" +certbot_cockpit_domain: "vh-01.lab.alexpires.me" diff --git a/ansible/playbook_certbot.yml b/ansible/playbook_certbot.yml index 10562ab..56e8b65 100644 --- a/ansible/playbook_certbot.yml +++ b/ansible/playbook_certbot.yml @@ -18,19 +18,35 @@ ansible.builtin.fail: msg: "CloudDNS nameserver is not set" - - name: Install Certbot and dependencies + - name: Remove Certbot installed via package manager (Debian/Ubuntu) + when: ansible_os_family == "Debian" + ansible.builtin.apt: + name: + - certbot + state: absent + + - name: Remove Certbot installed via package manager (RedHat/CentOS) when: ansible_os_family == "RedHat" ansible.builtin.dnf: name: - certbot - state: present + state: absent - name: Install cloudDNS plugin dependencies ansible.builtin.pip: name: + - certbot - certbot-dns-cloudns state: present + - name: Ensure CloudDNS credentials directory exists + ansible.builtin.file: + path: "/etc/letsencrypt" + state: directory + owner: root + group: root + mode: '0700' + - name: Create ClouDNS credentials file ansible.builtin.copy: dest: "{{ cloudns_credentials_path }}" @@ -59,3 +75,18 @@ job: "certbot renew --quiet" minute: 0 hour: 3 + + - name: Setup cockpit certifcate (if defined) + when: certbot_cockpit_domain is defined + block: + - name: Create symlink for Cockpit certificate + ansible.builtin.file: + src: "/etc/letsencrypt/live/{{ certbot_cockpit_domain }}/fullchain.pem" + dest: "/etc/cockpit/ws-certs.d/99-letsencrypt-cert.cert" + state: link + + - name: Create symlink for Cockpit private key + ansible.builtin.file: + src: "/etc/letsencrypt/live/{{ certbot_cockpit_domain }}/privkey.pem" + dest: "/etc/cockpit/ws-certs.d/99-letsencrypt-cert.key" + state: link diff --git a/inventory/hosts b/inventory/hosts index 09a160e..d48fdcd 100644 --- a/inventory/hosts +++ b/inventory/hosts @@ -18,6 +18,10 @@ dev-01.lab.alexpires.me vh-01.lab.alexpires.me gpu-01.lab.alexpires.me +[certbot] +vh-01.lab.alexpires.me +gpu-01.lab.alexpires.me + [public] prod-01.alexpires.me diff --git a/scripts/organize_images.sh b/scripts/organize_images.sh new file mode 100644 index 0000000..6782af1 --- /dev/null +++ b/scripts/organize_images.sh @@ -0,0 +1,182 @@ +#!/bin/bash + +# Parse arguments +DRY_RUN=false +INPUT_DIR="" +OUTPUT_DIR="" + +while [[ $# -gt 0 ]]; do + case $1 in + --dry-run|-n) + DRY_RUN=true + shift + ;; + --input|-i) + INPUT_DIR="$2" + shift 2 + ;; + --output|-o) + OUTPUT_DIR="$2" + shift 2 + ;; + --help|-h) + echo "Usage: $0 --input DIR --output DIR [OPTIONS]" + echo "" + echo "Required:" + echo " --input, -i DIR Input directory containing images" + echo " --output, -o DIR Output directory for organized images" + echo "" + echo "Options:" + echo " --dry-run, -n Preview changes without moving files" + echo " --help, -h Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Validate required arguments +if [[ -z "$INPUT_DIR" ]]; then + echo "Error: --input directory is required" + echo "Use --help for usage information" + exit 1 +fi + +if [[ -z "$OUTPUT_DIR" ]]; then + echo "Error: --output directory is required" + echo "Use --help for usage information" + exit 1 +fi + +# Check if input directory exists +if [[ ! -d "$INPUT_DIR" ]]; then + echo "Error: Input directory '$INPUT_DIR' does not exist" + exit 1 +fi + +# Check if exiftool is installed +if ! command -v exiftool &> /dev/null; then + echo "Error: exiftool is not installed. Please install it first." + echo " Ubuntu/Debian: sudo apt-get install libimage-exiftool-perl" + echo " macOS: brew install exiftool" + exit 1 +fi + +# Create output directory (only if not dry-run) +if [[ "$DRY_RUN" == false ]]; then + mkdir -p "$OUTPUT_DIR" +fi + +# Counters +processed=0 +unknown=0 +failed=0 + +if [[ "$DRY_RUN" == true ]]; then + echo "🔍 DRY RUN MODE - No files will be moved" +fi +echo "Starting image organization..." +echo "Input directory: $INPUT_DIR" +echo "Output directory: $OUTPUT_DIR" +echo "" + +# Process each image +# Using -iname for case-insensitive matching (matches .jpg, .JPG, .Jpg, etc.) +while IFS= read -r image; do + echo "Processing: $(basename "$image")" + + # Extract creation date (YYYYMMDD) + # Try DateTimeOriginal first (most common for photos), then CreateDate, then ModifyDate + creation_date=$(exiftool -DateTimeOriginal -d "%Y%m%d" -s3 "$image" 2>/dev/null | head -n 1 | tr -d ' ') + + if [[ -z "$creation_date" ]] || [[ ! "$creation_date" =~ ^[0-9]{8}$ ]]; then + creation_date=$(exiftool -CreateDate -d "%Y%m%d" -s3 "$image" 2>/dev/null | head -n 1 | tr -d ' ') + fi + + if [[ -z "$creation_date" ]] || [[ ! "$creation_date" =~ ^[0-9]{8}$ ]]; then + creation_date=$(exiftool -ModifyDate -d "%Y%m%d" -s3 "$image" 2>/dev/null | head -n 1 | tr -d ' ') + fi + + # Check if date was extracted successfully + if [[ -z "$creation_date" ]] || [[ ! "$creation_date" =~ ^[0-9]{8}$ ]]; then + echo " ⚠ Warning: Could not extract valid date, moving to Unknown folder" + target_dir="$OUTPUT_DIR/Unknown" + destination="$target_dir/$(basename "$image")" + + # Check if destination file already exists + if [[ -f "$destination" ]]; then + # Add timestamp to avoid overwrite + filename=$(basename "$image") + extension="${filename##*.}" + basename_no_ext="${filename%.*}" + destination="$target_dir/${basename_no_ext}_$(date +%s).$extension" + echo " ⚠ File exists, renaming to: $(basename "$destination")" + fi + + if [[ "$DRY_RUN" == true ]]; then + echo " → Would move to: Unknown/$(basename "$destination")" + else + mkdir -p "$target_dir" + if mv "$image" "$destination" 2>/dev/null; then + echo " ✓ Moved to: Unknown/" + else + echo " ✗ Failed to move file" + failed=$((failed + 1)) + continue + fi + fi + unknown=$((unknown + 1)) + continue + fi + + # Extract year, month, day + year="${creation_date:0:4}" + month="${creation_date:4:2}" + day="${creation_date:6:2}" + + # Create subdirectories (YYYY/MM/DD) + target_dir="$OUTPUT_DIR/$year/$month/$day" + + # Check if destination file already exists + destination="$target_dir/$(basename "$image")" + if [[ -f "$destination" ]]; then + # Add timestamp to avoid overwrite + filename=$(basename "$image") + extension="${filename##*.}" + basename_no_ext="${filename%.*}" + destination="$target_dir/${basename_no_ext}_$(date +%s).$extension" + echo " ⚠ File exists, renaming to: $(basename "$destination")" + fi + + if [[ "$DRY_RUN" == true ]]; then + # Dry run: just show what would happen + echo " → Would move to: $year/$month/$day/$(basename "$destination")" + processed=$((processed + 1)) + else + # Actually perform the move + mkdir -p "$target_dir" + if mv "$image" "$destination" 2>/dev/null; then + echo " ✓ Moved to: $year/$month/$day/" + processed=$((processed + 1)) + else + echo " ✗ Failed to move file" + failed=$((failed + 1)) + fi + fi +done < <(find "$INPUT_DIR" -type f \( -iname "*.jpg" -o -iname "*.jpeg" -o -iname "*.png" -o -iname "*.gif" -o -iname "*.bmp" -o -iname "*.tiff" -o -iname "*.tif" -o -iname "*.heic" -o -iname "*.heif" -o -iname "*.raw" -o -iname "*.cr2" -o -iname "*.nef" -o -iname "*.arw" -o -iname "*.dng" -o -iname "*.webp" \)) + +echo "" +if [[ "$DRY_RUN" == true ]]; then + echo "DRY RUN Summary:" + echo " Would process: $processed" + echo " Would move to Unknown: $unknown" +else + echo "Summary:" + echo " Processed: $processed" + echo " Moved to Unknown: $unknown" + echo " Failed: $failed" +fi diff --git a/scripts/organize_videos.sh b/scripts/organize_videos.sh new file mode 100644 index 0000000..7f104f3 --- /dev/null +++ b/scripts/organize_videos.sh @@ -0,0 +1,173 @@ +#!/bin/bash + +# Parse arguments +DRY_RUN=false +INPUT_DIR="" +OUTPUT_DIR="" + +while [[ $# -gt 0 ]]; do + case $1 in + --dry-run|-n) + DRY_RUN=true + shift + ;; + --input|-i) + INPUT_DIR="$2" + shift 2 + ;; + --output|-o) + OUTPUT_DIR="$2" + shift 2 + ;; + --help|-h) + echo "Usage: $0 --input DIR --output DIR [OPTIONS]" + echo "" + echo "Required:" + echo " --input, -i DIR Input directory containing videos" + echo " --output, -o DIR Output directory for organized videos" + echo "" + echo "Options:" + echo " --dry-run, -n Preview changes without moving files" + echo " --help, -h Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Validate required arguments +if [[ -z "$INPUT_DIR" ]]; then + echo "Error: --input directory is required" + echo "Use --help for usage information" + exit 1 +fi + +if [[ -z "$OUTPUT_DIR" ]]; then + echo "Error: --output directory is required" + echo "Use --help for usage information" + exit 1 +fi + +# Check if input directory exists +if [[ ! -d "$INPUT_DIR" ]]; then + echo "Error: Input directory '$INPUT_DIR' does not exist" + exit 1 +fi + +# Check if exiftool is installed +if ! command -v exiftool &> /dev/null; then + echo "Error: exiftool is not installed. Please install it first." + echo " Ubuntu/Debian: sudo apt-get install libimage-exiftool-perl" + echo " macOS: brew install exiftool" + exit 1 +fi + +# Create output directory (only if not dry-run) +if [[ "$DRY_RUN" == false ]]; then + mkdir -p "$OUTPUT_DIR" +fi + +# Counters +processed=0 +skipped=0 +failed=0 + +if [[ "$DRY_RUN" == true ]]; then + echo "🔍 DRY RUN MODE - No files will be moved" +fi +echo "Starting video organization..." +echo "Input directory: $INPUT_DIR" +echo "Output directory: $OUTPUT_DIR" +echo "" + +# Process each video +# Using -iname for case-insensitive matching (matches .mp4, .MP4, .Mp4, etc.) +while IFS= read -r video; do + echo "Processing: $(basename "$video")" + + # Extract creation date (YYYYMMDD) + creation_date=$(exiftool -MediaCreateDate -d "%Y%m%d" -s3 "$video" 2>/dev/null | head -n 1 | tr -d ' ') + + # Check if date was extracted successfully + if [[ -z "$creation_date" ]] || [[ ! "$creation_date" =~ ^[0-9]{8}$ ]]; then + echo " ⚠ Warning: Could not extract valid date, moving to Unknown folder" + target_dir="$OUTPUT_DIR/Unknown" + destination="$target_dir/$(basename "$video")" + + # Check if destination file already exists + if [[ -f "$destination" ]]; then + # Add timestamp to avoid overwrite + filename=$(basename "$video") + extension="${filename##*.}" + basename_no_ext="${filename%.*}" + destination="$target_dir/${basename_no_ext}_$(date +%s).$extension" + echo " ⚠ File exists, renaming to: $(basename "$destination")" + fi + + if [[ "$DRY_RUN" == true ]]; then + echo " → Would move to: Unknown/$(basename "$destination")" + else + mkdir -p "$target_dir" + if mv "$video" "$destination" 2>/dev/null; then + echo " ✓ Moved to: Unknown/" + else + echo " ✗ Failed to move file" + failed=$((failed + 1)) + continue + fi + fi + skipped=$((skipped + 1)) + continue + fi + + # Extract year, month, day + year="${creation_date:0:4}" + month="${creation_date:4:2}" + day="${creation_date:6:2}" + + # Create subdirectories (YYYY/MM/DD) + target_dir="$OUTPUT_DIR/$year/$month/$day" + + # Check if destination file already exists + destination="$target_dir/$(basename "$video")" + if [[ -f "$destination" ]]; then + # Add timestamp to avoid overwrite + filename=$(basename "$video") + extension="${filename##*.}" + basename_no_ext="${filename%.*}" + destination="$target_dir/${basename_no_ext}_$(date +%s).$extension" + echo " ⚠ File exists, renaming to: $(basename "$destination")" + fi + + if [[ "$DRY_RUN" == true ]]; then + # Dry run: just show what would happen + echo " → Would move to: $year/$month/$day/$(basename "$destination")" + processed=$((processed + 1)) + else + # Actually perform the move + mkdir -p "$target_dir" + if mv "$video" "$destination" 2>/dev/null; then + echo " ✓ Moved to: $year/$month/$day/" + processed=$((processed + 1)) + else + echo " ✗ Failed to move file" + failed=$((failed + 1)) + fi + fi +done < <(find "$INPUT_DIR" -type f \( -iname "*.mp4" -o -iname "*.mov" -o -iname "*.mkv" -o -iname "*.avi" -o -iname "*.wmv" -o -iname "*.flv" -o -iname "*.webm" \)) + +echo "" +if [[ "$DRY_RUN" == true ]]; then + echo "DRY RUN Summary:" + echo " Would process: $processed" + echo " Would move to Unknown: $skipped" +else + echo "Summary:" + echo " Processed: $processed" + echo " Moved to Unknown: $skipped" + echo " Failed: $failed" +fi \ No newline at end of file diff --git a/terraform/apps/dev-01/apps.tf b/terraform/apps/dev-01/apps.tf index 6fec830..d4c6da7 100644 --- a/terraform/apps/dev-01/apps.tf +++ b/terraform/apps/dev-01/apps.tf @@ -35,7 +35,7 @@ module "nextcloud" { source = "../../modules/apps/nextcloud" fqdn = local.nextcloud_fqdn - persistent_folder = local.persistent_folder + nextcloud_data = local.nextcloud_data app_config = local.nextcloud_config db_credentials_are_secrets = true diff --git a/terraform/apps/dev-01/config.tf b/terraform/apps/dev-01/config.tf index f51d47f..febf817 100644 --- a/terraform/apps/dev-01/config.tf +++ b/terraform/apps/dev-01/config.tf @@ -1,7 +1,7 @@ locals { # storage - persistent_folder = "/srv" + persistent_folder = "/mnt/srv" lab_domain = "lab.alexpires.me" internal_issuer = "internal-ca" @@ -14,7 +14,7 @@ locals { samba_shares = [ { name = "Storage" - path = "/mnt/storage" + path = "/mnt/md0/storage" comment = "Main Storage" } ] @@ -40,7 +40,7 @@ locals { { name = "vh-01" type = "A" - content = "10.19.4.109" + content = "10.19.4.254" }, { name = "gpu-01" @@ -176,8 +176,9 @@ locals { nextcloud_fqdn = "cloud.alexpires.me" nextcloud_db_passwd = var.nextcloud_db_passwd nextcloud_db_root_password = var.nextcloud_db_root_password - nextcloud_db_path = "${local.persistent_folder}/mysql/nextcloud" - backup_folder = "/mnt/usb/backups/nextcloud" + nextcloud_db_path = "/mnt/md0/nextcloud.db" + nextcloud_backup = "/mnt/md0/nextcloud.backups" + nextcloud_data = "/mnt/md0/nextcloud.files" nextcloud_config = { mail_from = "cloud" mail_domain = "alexpires.me" diff --git a/terraform/apps/dev-01/databases.tf b/terraform/apps/dev-01/databases.tf index 6cecc58..59060d5 100644 --- a/terraform/apps/dev-01/databases.tf +++ b/terraform/apps/dev-01/databases.tf @@ -10,7 +10,7 @@ module "nextcloud_db" { cron_schedule = local.backup_cron_schedule database_folder = local.nextcloud_db_path - backup_folder = local.backup_folder + backup_folder = local.nextcloud_backup tls_enabled = true issuer = local.internal_issuer diff --git a/terraform/modules/apps/nextcloud/config.tf b/terraform/modules/apps/nextcloud/config.tf index 2e402b8..23fef37 100644 --- a/terraform/modules/apps/nextcloud/config.tf +++ b/terraform/modules/apps/nextcloud/config.tf @@ -1,6 +1,4 @@ locals { - persistent_folder = var.persistent_folder - database_credentials = { MYSQL_USER = { key = "username" @@ -63,9 +61,6 @@ locals { } }, var.db_credentials_are_secrets ? local.database_credentials : {}) - - nextcloud_data = "${local.persistent_folder}/nextcloud" - apache_confs = { for f in fileset("${path.module}/resources/apache2/conf-enabled/", "*.conf") : "${f}" => file("${path.module}/resources/apache2/conf-enabled//${f}") diff --git a/terraform/modules/apps/nextcloud/main.tf b/terraform/modules/apps/nextcloud/main.tf index f3add72..8b565e5 100644 --- a/terraform/modules/apps/nextcloud/main.tf +++ b/terraform/modules/apps/nextcloud/main.tf @@ -229,7 +229,7 @@ resource "kubernetes_deployment" "nextcloud_apache" { volume { name = "html-data" host_path { - path = local.nextcloud_data + path = var.nextcloud_data } } volume { diff --git a/terraform/modules/apps/nextcloud/variables.tf b/terraform/modules/apps/nextcloud/variables.tf index e93ae7b..b77030c 100644 --- a/terraform/modules/apps/nextcloud/variables.tf +++ b/terraform/modules/apps/nextcloud/variables.tf @@ -1,4 +1,4 @@ -variable "persistent_folder" { +variable "nextcloud_data" { description = "The path to the persistent folder" type = string }