Improved workflows and actions

.github/workflows/actions-validate.yml

@@ -0,0 +1,31 @@
+name: Ansible Apply Playbook Test
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - .github/workflows/actions/**
+
+jobs:
+  test-setup-ansible-actions:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup environment (TEST)
+        uses: ./.github/actions/setup-env
+
+      - name: Setup Ansible environment (TEST)
+        uses: ./.github/actions/setup-ansible-env
+
+  test-setup-terraform-actions:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup environment (TEST)
+        uses: ./.github/actions/setup-env
+
+      - name: Setup Terraform environment (TEST)
+        uses: ./.github/actions/setup-terraform-env
+
+      - name: Setup Kubernetes Environment (TEST)
+        uses: ./.github/actions/setup-k8s-env

.github/workflows/actions/setup-ansible-env.yml

@@ -0,0 +1,57 @@
+name: Setup Ansible Environment
+description: Prepare the environment for A13labs CI/CD pipelines
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Python and required packages
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y python3-pip
+        python3 -m pip install -r requirements.txt
+
+    - name: Setup SSH Key
+      env:
+        VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
+      run: |
+        sectool ssh unlock ansible
+        mkdir -p ~/.ssh
+        chmod 700 ~/.ssh
+        cp ssh-keys/ansible/id_ecdsa ~/.ssh/id_ecdsa
+        chmod 600 ~/.ssh/id_ecdsa
+
+    - name: Read SSH Trusted Hosts keys
+      env:
+        TRUSTED_HOSTS: ${{ secrets.TRUSTED_HOSTS }}
+      run: |
+        mkdir -p ~/.ssh
+        for HOST in $TRUSTED_HOSTS; do
+          ssh-keyscan $HOST >> ~/.ssh/known_hosts
+        done
+        chmod 600 ~/.ssh/known_hosts
+
+    - name: Enable SSH host key algorithms
+      run: |
+        echo "Host *" >> ~/.ssh/config
+        echo "  HostKeyAlgorithms +sk-ecdsa-sha2-nistp256@openssh.com, sk-ssh-ed25519@openssh.com"
+        chmod 600 ~/.ssh/config
+
+    - name: Start SSH Agent
+      env:
+        VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+      run: |
+        ssh-agent -a $SSH_AUTH_SOCK > /dev/null
+        echo 'sectool vault get SSH_PASSWORD' > ~/.ssh_askpass && chmod +x ~/.ssh_askpass
+        DISPLAY=None SSH_ASKPASS=~/.ssh_askpass ssh-add ~/.ssh/id_ecdsa < /dev/null
+        rm ~/.ssh_askpass
+
+    - name: Check SSH Connection to remote hosts
+      env:
+        TRUSTED_HOSTS: ${{ secrets.TRUSTED_HOSTS }}
+        ANSIBLE_USER: ${{ secrets.ANSIBLE_USER }}
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+      run: |
+        for HOST in $TRUSTED_HOSTS; do
+          ssh $ANSIBLE_USER@$HOST echo "SSH connection to $HOST successful"
+        done

.github/workflows/actions/setup-env.yml

@@ -0,0 +1,18 @@
+name: Setup Environment
+description: Prepare the environment for A13labs CI/CD pipelines
+
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Get changed files
+      id: get_changed_files
+      run: |
+        # Get the list of changed files
+        CHANGED_FILES=$(git diff --name-only ${{ github.event.before }} ${{ github.sha }})
+        echo "CHANGED_FILES=$CHANGED_FILES" >> $GITHUB_ENV
+
+    - name: Setup Sectool
+      uses: a13labs/setup-sectool@v1

.github/workflows/actions/setup-k8s-env.yml

@@ -0,0 +1,41 @@
+name: Setup K8S Environment
+description: Prepare the environment for A13labs CI/CD pipelines
+inputs:
+  k8s-target:
+    description: "The target host to deploy the K8S applications"
+    default: "microk8s"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Copy Kube Config from remote host
+      env:
+        VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
+        ANSIBLE_USER: ${{ secrets.ANSIBLE_USER }}
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+      run: |
+        mkdir -p ~/.kube
+        ansible -i ansible/inventory/hosts -u $ANSIBLE_USER -m fetch -a "src=/home/$ANSIBLE_USER/.kube/config dest=~/.kube/config flat=yes" {{ inputs.k8s-target }}
+
+    - name: Terraform Init
+      env:
+        VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
+      run: |
+        cd terraform/k8sapps
+        sectool exec terraform init
+
+    - name: Terraform Apply
+      env:
+        VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
+        ANSIBLE_USER: ${{ secrets.ANSIBLE_USER }}
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+      run: |
+        # Get Hostname and IP address of the target
+        ANSIBLE_HOST=$(ansible -i ansible/inventory/hosts -u $ANSIBLE_USER -m shell -a "hostname" {{ inputs.k8s-target }} | grep -oP '(\w+\.){2}\w+' | head -1)
+        IP_ADDR=$(ansible -i ansible/inventory/hosts -u $ANSIBLE_USER -m shell -a "curl -s ifconfig.me" {{ inputs.k8s-target }} | grep -oP '(\d+\.){3}\d+')
+
+        # Forward the k8s API server port to localhost
+        ssh -f -N -L 16443:localhost:16443 $ANSIBLE_USER@$ANSIBLE_HOST
+
+        # Override the kube config file to use localhost
+        sed -i "s/server: https:\/\/$IP_ADDR:16443/server: https:\/\/localhost:16443/g" ~/.kube/config

.github/workflows/actions/setup-terraform-env.yml

@@ -0,0 +1,10 @@
+name: Setup Terraform Environment
+description: Prepare the environment for A13labs CI/CD pipelines
+
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@v1
+      with:
+        terraform_version: 1.9.6

.github/workflows/ansible-apply-playbook.yml

@@ -0,0 +1,30 @@
+name: Ansible Validate Playbook on Pull Request
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "ansible/playbook_*.yml"
+
+jobs:
+  ansible-apply:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
+
+      - name: Setup Ansible environment
+        uses: ./.github/actions/setup-ansible-env
+
+      - name: Apply the playbook that triggered the workflow
+        env:
+          VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
+          ANSIBLE_USER: ${{ secrets.ANSIBLE_USER }}
+          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+        run: |
+          cd ansible
+          for PLAYBOOK in $(echo $CHANGED_FILES | grep -oP 'playbook_\K\w+'); do
+            ansible-playbook -u $ANSIBLE_USER playbook_$PLAYBOOK.yml
+          done

.github/workflows/ansible-apply.yaml

@@ -1,90 +0,0 @@
-name: Ansible Apply
-
-on:
-  schedule:
-    - cron: "0 2 * * 0"
-  push:
-    branches:
-      - main
-    paths:
-      - "ansible/**"
-      - .github/workflows/ansible-apply.yaml
-
-jobs:
-  ansible-apply:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Setup Sectool
-        uses: a13labs/setup-sectool@v1
-
-      - name: Install Python and required packages
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y python3-pip
-          python3 -m pip install -r requirements.txt
-
-      - name: Setup SSH Key
-        env:
-          VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
-        run: |
-          sectool ssh unlock ansible
-          mkdir -p ~/.ssh
-          chmod 700 ~/.ssh
-          cp ssh-keys/ansible/id_ecdsa ~/.ssh/id_ecdsa
-          chmod 600 ~/.ssh/id_ecdsa
-
-      - name: Read SSH Trusted Hosts keys
-        env:
-          TRUSTED_HOSTS: ${{ secrets.TRUSTED_HOSTS }}
-        run: |
-          mkdir -p ~/.ssh
-          for HOST in $TRUSTED_HOSTS; do
-            ssh-keyscan $HOST >> ~/.ssh/known_hosts
-          done
-          chmod 600 ~/.ssh/known_hosts
-
-      - name: Enable SSH host key algorithms
-        run: |
-          echo "Host *" >> ~/.ssh/config
-          echo "  HostKeyAlgorithms +sk-ecdsa-sha2-nistp256@openssh.com, sk-ssh-ed25519@openssh.com"
-          chmod 600 ~/.ssh/config
-
-      - name: Start SSH Agent
-        env:
-          VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-        run: |
-          ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-          echo 'sectool vault get SSH_PASSWORD' > ~/.ssh_askpass && chmod +x ~/.ssh_askpass
-          DISPLAY=None SSH_ASKPASS=~/.ssh_askpass ssh-add ~/.ssh/id_ecdsa < /dev/null
-          rm ~/.ssh_askpass
-
-      - name: Check SSH Connection to remote hosts
-        env:
-          TRUSTED_HOSTS: ${{ secrets.TRUSTED_HOSTS }}
-          ANSIBLE_USER: ${{ secrets.ANSIBLE_USER }}
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-        run: |
-          for HOST in $TRUSTED_HOSTS; do
-            ssh $ANSIBLE_USER@$HOST echo "SSH connection to $HOST successful"
-          done
-
-      - name: Apply Ansible Playbook
-        env:
-          VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
-          ANSIBLE_USER: ${{ secrets.ANSIBLE_USER }}
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-        run: |
-          cd ansible
-          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_login.yml
-          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_cis.yml --skip-tags roles::cis::suid
-          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_encryption.yml
-          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_microk8s.yml
-          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_gitea.yml
-          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_nextcloud.yml
-          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_wordpress.yml
-          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_m3uproxy.yml

.github/workflows/ansible-validate-on-pr.yml

@@ -1,9 +1,11 @@
-name: Ansible Validate
+name: Ansible Validate On Pull Request
 
 on:
   pull_request:
     branches:
       - main
+    paths:
+      - "ansible/**"
     types:
       - opened
       - synchronize
@@ -14,11 +16,8 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Checkout repository
+      - name: Setup environment
-        uses: actions/checkout@v2
+        uses: ./.github/actions/setup-env
-
-      - name: Setup Sectool
-        uses: a13labs/setup-sectool@v1
 
       - name: Install Python and required packages
         run: |

.github/workflows/ansible-validate-playbook-on-pr.yml

@@ -0,0 +1,34 @@
+name: Ansible Validate Playbook on Pull Request
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "ansible/playbook_*.yml"
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+jobs:
+  ansible-apply:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
+
+      - name: Setup Ansible environment
+        uses: ./.github/actions/setup-ansible-env
+
+      - name: Apply the playbook that triggered the workflow
+        env:
+          VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
+          ANSIBLE_USER: ${{ secrets.ANSIBLE_USER }}
+          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+        run: |
+          cd ansible
+          for PLAYBOOK in $(echo $CHANGED_FILES | grep -oP 'playbook_\K\w+'); do
+            ansible-playbook -u $ANSIBLE_USER playbook_$PLAYBOOK.yml -CD
+          done

.github/workflows/ansible-weekly-reconciliation.yml

@@ -0,0 +1,32 @@
+name: Weekly Reconciliation
+
+on:
+  schedule:
+    - cron: "0 2 * * 0"
+
+jobs:
+  ansible-apply:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
+
+      - name: Setup Ansible environment
+        uses: ./.github/actions/setup-ansible-env
+
+      - name: Apply Hardening Playbook
+        env:
+          VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
+          ANSIBLE_USER: ${{ secrets.ANSIBLE_USER }}
+          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+        run: |
+          cd ansible
+          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_login.yml
+          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_cis.yml --skip-tags roles::cis::suid
+          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_encryption.yml
+          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_microk8s.yml
+          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_gitea.yml
+          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_nextcloud.yml
+          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_wordpress.yml
+          sectool exec ansible-playbook -u $ANSIBLE_USER playbook_m3uproxy.yml

.github/workflows/terraform-iac-apply.yml

@@ -1,4 +1,4 @@
-name: Terraform Apply
+name: Terraform IAC Apply
 
 on:
   schedule:
@@ -8,23 +8,17 @@ on:
       - main
     paths:
       - "terraform/iac/**"
-      - .github/workflows/terraform-iac-apply.yaml
 
 jobs:
   terraform-apply:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Checkout repository
+      - name: Setup environment
-        uses: actions/checkout@v2
+        uses: ./.github/actions/setup-env
 
-      - name: Setup Terraform
+      - name: Setup Terraform Environment
-        uses: hashicorp/setup-terraform@v1
+        uses: ./.github/actions/setup-terraform-env
-        with:
-          terraform_version: 1.9.6
-
-      - name: Setup Sectool
-        uses: a13labs/setup-sectool@v1
 
       - name: Terraform Init
         env:

.github/workflows/terraform-iac-validate-on-pr.yml

@@ -1,4 +1,4 @@
-name: Terraform Validate
+name: Terraform IAC Validate on Pull Request
 
 on:
   pull_request:
@@ -6,7 +6,6 @@ on:
       - main
     paths:
       - "terraform/iac/**"
-      - .github/workflows/terraform-iac-validate.yaml
     types:
       - opened
       - synchronize
@@ -16,16 +15,11 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Checkout repository
+      - name: Setup environment
-        uses: actions/checkout@v2
+        uses: ./.github/actions/setup-env
 
-      - name: Setup Terraform
+      - name: Setup Terraform Environment
-        uses: hashicorp/setup-terraform@v1
+        uses: ./.github/actions/setup-terraform-env
-        with:
-          terraform_version: 1.9.6
-
-      - name: Setup Sectool
-        uses: a13labs/setup-sectool@v1
 
       - name: Terraform Init and Validate
         env:

.github/workflows/terraform-k8apps-apply.yaml

@@ -1,103 +0,0 @@
-name: Terraform Apply
-
-on:
-  push:
-    branches:
-      - main
-    paths:
-      - "terraform/k8sapps/**"
-      - .github/workflows/terraform-k8apps-apply.yaml
-
-jobs:
-  terraform-apply:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Setup Terraform
-        uses: hashicorp/setup-terraform@v1
-        with:
-          terraform_version: 1.9.6
-
-      - name: Setup Sectool
-        uses: a13labs/setup-sectool@v1
-
-      - name: Install Python and required packages
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y python3-pip
-          python3 -m pip install -r requirements.txt
-
-      - name: Setup SSH Key
-        env:
-          VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
-        run: |
-          sectool ssh unlock ansible
-          mkdir -p ~/.ssh
-          chmod 700 ~/.ssh
-          cp ssh-keys/ansible/id_ecdsa ~/.ssh/id_ecdsa
-          chmod 600 ~/.ssh/id_ecdsa
-
-      - name: Read SSH Trusted Hosts keys
-        env:
-          TRUSTED_HOSTS: ${{ secrets.TRUSTED_HOSTS }}
-        run: |
-          mkdir -p ~/.ssh
-          for HOST in $TRUSTED_HOSTS; do
-            ssh-keyscan $HOST >> ~/.ssh/known_hosts
-          done
-          chmod 600 ~/.ssh/known_hosts
-
-      - name: Start SSH Agent
-        env:
-          VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-        run: |
-          ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-          echo 'sectool vault get SSH_PASSWORD' > ~/.ssh_askpass && chmod +x ~/.ssh_askpass
-          DISPLAY=None SSH_ASKPASS=~/.ssh_askpass ssh-add ~/.ssh/id_ecdsa < /dev/null
-          rm ~/.ssh_askpass
-
-      - name: Check SSH Connection to remote hosts
-        env:
-          TRUSTED_HOSTS: ${{ secrets.TRUSTED_HOSTS }}
-          ANSIBLE_USER: ${{ secrets.ANSIBLE_USER }}
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-        run: |
-          for HOST in $TRUSTED_HOSTS; do
-            ssh $ANSIBLE_USER@$HOST echo "SSH connection to $HOST successful"
-          done
-
-      - name: Copy Kube Config from remote host
-        env:
-          VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
-          ANSIBLE_USER: ${{ secrets.ANSIBLE_USER }}
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-        run: |
-          mkdir -p ~/.kube
-          ansible -i ansible/inventory/hosts -u $ANSIBLE_USER -m fetch -a "src=/home/$ANSIBLE_USER/.kube/config dest=~/.kube/config flat=yes" microk8s
-
-      - name: Terraform Init
-        env:
-          VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
-        run: |
-          cd terraform/k8sapps
-          sectool exec terraform init
-
-      - name: Terraform Apply
-        env:
-          VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
-          ANSIBLE_USER: ${{ secrets.ANSIBLE_USER }}
-          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-          TRUSTED_HOSTS: ${{ secrets.TRUSTED_HOSTS }}
-        run: |
-          for HOST in $TRUSTED_HOSTS; do
-            ssh -f -N -L 16443:localhost:16443 $ANSIBLE_USER@$HOST
-            IP_ADDR=$(ssh $ANSIBLE_USER@$HOST "hostname -I | cut -d' ' -f1")
-            sed -i "s/server: https:\/\/$IP_ADDR:16443/server: https:\/\/localhost:16443/g" ~/.kube/config
-            cd terraform/k8sapps
-            sectool exec terraform apply --auto-approve
-            pkill -f "ssh -f -N -L 16443:localhost:16443 $ANSIBLE_USER@$HOST"
-          done

.github/workflows/terraform-microk8s-apply.yml

@@ -0,0 +1,42 @@
+name: Terraform K8S Apps apply
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "terraform/k8sapps/**"
+
+jobs:
+  terraform-apply:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
+
+      - name: Setup Ansible Environment
+        uses: ./.github/actions/setup-ansible-env
+
+      - name: Setup Terraform Environment
+        uses: ./.github/actions/setup-terraform-env
+
+      - name: Setup Kubernetes Environment
+        uses: ./.github/actions/setup-k8s-env
+
+      - name: Terraform Init
+        env:
+          VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
+        run: |
+          cd terraform/k8sapps
+          sectool exec terraform init
+
+      - name: Terraform Apply
+        env:
+          VAULT_MASTER_PASSWORD: ${{ secrets.VAULT_MASTER_PASSWORD }}
+          ANSIBLE_USER: ${{ secrets.ANSIBLE_USER }}
+          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+        run: |
+          # Apply the Terraform configuration
+          cd terraform/k8sapps
+          sectool exec terraform apply --auto-approve

.github/workflows/terraform-microk8s-validate.yml

@@ -1,4 +1,4 @@
-name: Terraform Validate
+name: Terraform K8s Apps Validate on Pull Request
 
 on:
   pull_request:
@@ -6,7 +6,6 @@ on:
       - main
     paths:
       - "terraform/k8sapps/**"
-      - .github/workflows/terraform-k8apps-validate.yaml
     types:
       - opened
       - synchronize
@@ -17,7 +16,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v1