revamp the github action workflows (#43)

HarshCasper · HarshCasper · commit 7354e0ba92ea · 2025-04-23T00:14:18.000+05:30
* fix the preview workflow

* fix

* fix

* test against a branch

* remove the branch

* fix integration test with auth token

* try running on macos

* setup docker on macos

* run macOS tests only on schedule

* remove cloud pods release workflow

* reform the cloud pods action

* remove keepalive workflow
diff --git a/.github/workflows/cloudpod_release.yml b/.github/workflows/cloudpod_release.yml
diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -22,15 +22,15 @@ on:
         type: choice
         options:
           - ubuntu-latest
-          - macos-latest
+          - macos-13
 
 
 jobs:
   run-it-tests-job:
     runs-on: ${{ inputs.runner-os || 'ubuntu-latest' }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.11
         id: setup-python
@@ -39,9 +39,9 @@ jobs:
           python-version: 3.11
 
       - name: Docker setup (macos only)
-        id: setup-docker-mac
         if: ${{ runner.os == 'macOS' }}
         run: |
+          brew install colima
           brew install docker
           colima start
 
@@ -50,14 +50,15 @@ jobs:
           pip install -r requirements-dev.txt
 
       - name: Start LocalStack
-        uses: LocalStack/setup-localstack@v0.2.2
+        uses: LocalStack/setup-localstack@v0.2.4
         with:
           image-tag: 'latest'
           use-pro: 'true'
           configuration: LS_LOG=trace
           install-awslocal: 'true'
         env:
-          LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }}
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
+
       - name: Build lambdas
         run: |
           bin/build_lambdas.sh
diff --git a/.github/workflows/preview_create.yml b/.github/workflows/preview_create.yml
@@ -7,7 +7,6 @@ on:
       - ./*.md
       - LICENSE
       - .circleci/*
-      - .gitlab-ci.yml
 
 jobs:
   test:
@@ -30,20 +29,16 @@ jobs:
           pip install awscli-local
 
       - name: Deploy Preview
-        uses: LocalStack/setup-localstack@v0.2.2
+        uses: LocalStack/setup-localstack@main
         env:
-          AWS_DEFAULT_REGION: us-east-1
-          AWS_REGION: us-east-1
-          AWS_ACCESS_KEY_ID: test
-          AWS_SECRET_ACCESS_KEY: test
-          LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }}
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           state-backend: ephemeral
           state-action: start
-          skip-ephemeral-stop: 'true'
           include-preview: 'true'
+          install-awslocal: 'true'
           preview-cmd: |
             # Add your custom deployment commands here. 
             # Below is an example for the Image resizer application.
-            bin/build_lambdas.sh && deployment/awslocal/deploy.sh
+            bin/build_lambdas.sh && deployment/awslocal/deploy.sh
diff --git a/.github/workflows/test_cloudpods.yml b/.github/workflows/test_cloudpods.yml
@@ -1,75 +1,111 @@
-name: Test Released Cloud Pods
+name: Create and Test LocalStack Cloud Pod
 
 on:
   schedule:
-    # “At 00:00 on Saturday.”
+    # At 00:00 on Saturday.
     - cron: "0 0 * * 6"
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
   workflow_dispatch:
 
 permissions:
   contents: write
+  actions: read
 
 jobs:
-  get-releases:
-    name: Retrieve Released Cloud Pods
+  create-pod:
+    name: Create Cloud Pod
     runs-on: ubuntu-latest
+    permissions:
+      actions: write
     outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      pod_artifact_name: cloud-pod-${{ github.run_id }}
     steps:
-    - id: set-matrix
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: |
-        output=$(gh api repos/$GITHUB_REPOSITORY/releases | jq -r '[.[] | select(.tag_name|startswith("v")|not) | .tag_name]')
-        output=$(echo $output | tr '\n' ' ')
-        echo "matrix=$output" >> $GITHUB_OUTPUT
-  
-  test-pod-release:
-    needs: get-releases
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix: 
-        tag: ${{ fromJson(needs.get-releases.outputs.matrix) }}
-    steps:
-      # checkout to run the tests later on
-      - name: Checkout
-        uses: actions/checkout@v3
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install Dependencies
+        run: |
+          pip install -r requirements-dev.txt --upgrade
+
+      - name: Start LocalStack
+        uses: LocalStack/setup-localstack@main
+        with:
+          use-pro: 'true'
+          install-awslocal: 'true'
+        env:
+          DEBUG: 1
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
+
+      - name: Deploy Infrastructure (Example)
+        run: |
+          bin/build_lambdas.sh && deployment/awslocal/deploy.sh
 
-      # Loading it manually as we're storing the state as a release and not an artifact
-      - name: Retrieve Pod
+      - name: Export LocalStack State (Cloud Pod)
+        id: export_state
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
         run: |
-          # TODO the download url seems to follow the pattern $GITHUB_REPOSITORY/releases/download/{TAG}/{ASSET_NAME}
-          # alternatively we can query the asset-id, and browser_download_url, but it seems like an overhead
-          # asset_id=$(gh api repos/$GITHUB_REPOSITORY/releases/tags/latest | jq -r '.assets[]' | jq --arg DB $DB -c 'select(.name=="release-pod-\( $DB ).zip") | .id)
-          # download_url=$(gh api repos/$GITHUB_REPOSITORY/releases/assets/$asset_id | jq -r ".browser_download_url")
-          download_url="https://github.com/$GITHUB_REPOSITORY/releases/download/${{ matrix.tag }}/release-pod.zip"
-          curl -L $download_url --output release-pod.zip
-          ls -la
+          echo "Exporting LocalStack state..."
+          localstack state export cloud-pod.zip
+          ls -la cloud-pod.zip # Verify the file exists
+          echo "pod_artifact_name=cloud-pod-${{ github.run_id }}" >> $GITHUB_OUTPUT
+
+      - name: Upload Cloud Pod Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ steps.export_state.outputs.pod_artifact_name }}
+          path: cloud-pod.zip
+          retention-days: 1
+
+  test-pod:
+    name: Test Cloud Pod
+    needs: create-pod
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.11'
 
+      - name: Install Test Dependencies
+        run: |
+          pip install -r requirements-dev.txt --upgrade
+
       - name: Start LocalStack
-        uses: LocalStack/setup-localstack@v0.2.2
+        uses: LocalStack/setup-localstack@main
         with:
-          image-tag: ${{ matrix.tag }}
           use-pro: 'true'
           install-awslocal: 'true'
         env:
           DEBUG: 1
           POD_LOAD_CLI_TIMEOUT: 300
-          LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }}
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
+
+      - name: Download Cloud Pod Artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ needs.create-pod.outputs.pod_artifact_name }}
 
-      - name: Inject Pod
+      - name: Inject Pod (Import State)
         env:
-          LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }}
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }}
         run: |
-          localstack state import release-pod.zip
+          echo "Importing LocalStack state from cloud-pod.zip..."
+          ls -la # Check if download was successful
+          localstack state import cloud-pod.zip
 
       - name: Run Tests
         env:
@@ -78,11 +114,10 @@ jobs:
           AWS_ACCESS_KEY_ID: test
           AWS_SECRET_ACCESS_KEY: test
         run: |
-          pip install -r requirements-dev.txt
           pytest tests
 
-      - name: Show Logs
-        if: failure()
+      - name: Show LocalStack Logs
+        if: always()
         run: |
           localstack logs
 
@@ -99,10 +134,18 @@ jobs:
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
 
-      - name: Prevent Workflows from getting Stale
-        if: always()
-        uses: gautamkrishnar/keepalive-workflow@v1
+      - name: Generate a Diagnostic Report
+        if: failure()
+        run: |
+          curl -s localhost:4566/_localstack/diagnose | gzip -cf > diagnose.json.gz
+
+      - name: Upload the Diagnostic Report
+        if: failure()
+        uses: actions/upload-artifact@v4
         with:
-          # this message should prevent automatic triggering of workflows
-          # see https://docs.github.com/en/actions/managing-workflow-runs/skipping-workflow-runs
-          commit_message: "[skip ci] Automated commit by Keepalive Workflow to keep the repository active"
+          name: diagnose.json.gz
+          path: ./diagnose.json.gz
+
+      - name: Prevent Workflow from becoming Stale
+        if: always() && github.ref == 'refs/heads/main'
+        uses: liskin/gh-workflow-keepalive@v1
