add checkpoint storage

Author: ajithvcoder

.github/workflows/ec2-pipeline.yml

@@ -53,22 +53,14 @@ jobs:
       # - uses: actions/checkout@v2
 
       - name: Display CUDA Version
-        run: |
-          if command -v nvcc &> /dev/null; then
-            echo "CUDA Version:"
-            nvcc --version || true
-          else
-            echo "nvcc not found. Ensure CUDA is installed."
-          fi
+        run: |         
+          echo "CUDA Version:"
+          nvcc --version || true
 
       - name: Display cuDNN Version
         run: |
-          if [ -f /usr/local/cuda/include/cudnn_version.h ]; then
-            echo "cuDNN Version:"
-            cat /usr/local/cuda/include/cudnn_version.h | grep CUDNN_MAJOR -A 2 || true
-          else
-            echo "cuDNN not found. Ensure cuDNN is installed."
-          fi
+          echo "cuDNN Version:"
+          cat /usr/local/cuda/include/cudnn_version.h | grep CUDNN_MAJOR -A 2 || true
 
       - name: Verify EC2 Instance
         run: |
@@ -153,7 +145,11 @@ jobs:
 
       - name: Run DVC commands in container
         run: |
+          mkdir model_storage
+          touch best_model_checkpoint.txt
           docker run --rm --gpus=all \
+            -v model_storage:/workspace/model_storage \
+            -v best_model_checkpoint.txt:/workspace/best_model_checkpoint.txt
             -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
             -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
             -e AWS_DEFAULT_REGION=${{ secrets.AWS_REGION }} \
@@ -173,6 +169,24 @@ jobs:
           # # Stop the container after retrieving logs
           # docker stop $CONTAINER_ID
 
+      - name: Read best checkpoint file name
+        id: read_checkpoint
+        run: |
+          checkpoint_file=$(head -n 1 best_model_checkpoint.txt)
+          echo "CHECKPOINT_FILE=$checkpoint_file" >> $GITHUB_ENV
+
+      - name: Get latest commit ID
+        id: get_commit_id
+        run: echo "COMMIT_ID=$(git rev-parse HEAD)" >> $GITHUB_ENV
+
+      - name: Upload checkpoint to S3
+        run: |
+          checkpoint_path="${{ env.CHECKPOINT_FILE }}"  # Use the checkpoint path from the file
+          bucket_name="mybucket-emlo-mumbai/session-08-checkpoint/"  # Change to your S3 bucket name
+          s3_key="session-08-checkpoint/${{ env.COMMIT_ID }}/$(basename "$checkpoint_path")"  # Define S3 key
+          echo "Uploading $checkpoint_path to s3://$bucket_name/$s3_key"
+          aws s3 cp "$checkpoint_path" "s3://$bucket_name/$s3_key"
+
       - name: Clean previous images and containers
         run: |
           docker system prune -f