add files

Author: ajithvcoder

.github/workflows/ec2-pipeline.yml

@@ -147,16 +147,15 @@ jobs:
         run: |
           mkdir -p model_storage
           docker run --rm --gpus=all \
-            -v model_storage:/workspace/model_storage \
+            -v $(pwd):/workspace \
             -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
             -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
             -e AWS_DEFAULT_REGION=${{ secrets.AWS_REGION }} \
             ${{ secrets.AWS_ECR_LOGIN_URI }}/${{ secrets.ECR_REPOSITORY_NAME }}:latest \
             /bin/bash -c "
               dvc pull -r myremote && \
               mkdir -p model_storage && \
-              dvc repro -f && \
-              cp best_model_checkpoint.txt model_storage/
+              dvc repro -f 
             "
 
           # # Wait a moment to ensure the container has started

Dockerfile

@@ -1,4 +1,4 @@
-FROM pytorch/pytorch:2.4.0-cuda12.4-cudnn9-runtime
+FROM pytorch/pytorch:2.3.1-cuda11.8-cudnn8-runtime
 
 WORKDIR /workspace
 COPY . .

configs/callbacks/model_checkpoint.yaml

@@ -3,7 +3,7 @@
 model_checkpoint:
   _target_: src.train.CustomModelCheckpiont
   dirpath: null # directory to save the model file
-  filename: "/home/runner/work/emlo4-session-07-ajithvcoder/emlo4-session-07-ajithvcoder/model_storage/epoch-checkpoint.ckpt.ckpt" # checkpoint filename
+  filename: "/workspace/model_storage/epoch-checkpoint.ckpt.ckpt" # checkpoint filename
   monitor: null # name of the logged metric which determines when model is improving
   verbose: False # verbosity mode
   save_last: null # additionally always save an exact copy of the last checkpoint to a file last.ckpt

configs/experiment/catdog_ex.yaml

@@ -46,7 +46,7 @@ callbacks:
     mode: "max"
     save_top_k: 1
     save_last: True
-    filename: "/home/runner/work/emlo4-session-07-ajithvcoder/emlo4-session-07-ajithvcoder/model_storage/epoch-checkpoint"
+    filename: "/workspace/model_storage/epoch-checkpoint"
 
   early_stopping:
     monitor: "val_acc"

configs/experiment/dogbreed_ex.yaml

@@ -35,7 +35,7 @@ callbacks:
     mode: "max"
     save_top_k: 1
     save_last: True
-    filename: "/home/runner/work/emlo4-session-07-ajithvcoder/emlo4-session-07-ajithvcoder/model_storage/epoch-checkpoint.ckpt"
+    filename: "/workspace/model_storage/epoch-checkpoint.ckpt"
 
   early_stopping:
     monitor: "val_acc"

configs/experiment/dogbreed_ex_train.yaml

@@ -35,7 +35,7 @@ callbacks:
     mode: "max"
     save_top_k: 1
     save_last: True
-    filename: "/home/runner/work/emlo4-session-07-ajithvcoder/emlo4-session-07-ajithvcoder/emlo4-session-06-ajithvcoder/model_storage/epoch-checkpoint"
+    filename: "/workspace/emlo4-session-06-ajithvcoder/model_storage/epoch-checkpoint"
 
   early_stopping:
     monitor: "val_acc"

configs/infer.yaml

@@ -31,4 +31,4 @@ callbacks:
     mode: "max"
     save_top_k: 1
     save_last: True
-    filename: "/home/runner/work/emlo4-session-07-ajithvcoder/emlo4-session-07-ajithvcoder/model_storage/epoch-checkpoint.ckpt.ckpt"
+    filename: "/workspace/model_storage/epoch-checkpoint.ckpt.ckpt"

dvc.lock

@@ -5,8 +5,8 @@ stages:
     deps:
     - path: configs/experiment/catdog_ex.yaml
       hash: md5
-      md5: d9430d3b364df5d20fc94f90d8cc6ad1
-      size: 1280
+      md5: 959c64c29a91c7f7b4448e0c5cbc54f1
+      size: 1215
     - path: data/cats_and_dogs_filtered
       hash: md5
       md5: c57d92307d2679437e80f6682cfaf521.dir
@@ -23,8 +23,8 @@ stages:
     deps:
     - path: configs/experiment/catdog_ex.yaml
       hash: md5
-      md5: 912d97cb77912129ba1ae8427cddc766
-      size: 1240
+      md5: 959c64c29a91c7f7b4448e0c5cbc54f1
+      size: 1215
     - path: src/eval.py
       hash: md5
       md5: 47a9e50a0213ee71ff67a97b71d1ca6e
@@ -34,8 +34,8 @@ stages:
     deps:
     - path: configs/experiment/catdog_ex.yaml
       hash: md5
-      md5: 912d97cb77912129ba1ae8427cddc766
-      size: 1240
+      md5: 959c64c29a91c7f7b4448e0c5cbc54f1
+      size: 1215
     - path: src/infer.py
       hash: md5
       md5: 1b360e17f7740c9bd6eba9c52e0dc5d2

scripts/multirun_metrics_fetch.py

@@ -62,7 +62,7 @@ def get_hyperparams(log_dir, timestamp):
                                     "patch_size": values.get('patch_size'), "embed_dim": values.get('embed_dim')}
     return extract_hyperparams
 
-def main():
+def main_run():
     log_dir = 'logs'
     timestamp = get_latest_timestamp(os.path.join(log_dir, 'train/multiruns'))
 
@@ -177,5 +177,39 @@ def main():
     with open('best_model_checkpoint.txt', 'w') as f:
         f.write(f"./model_storage/epoch-checkpoint_patch_size-{hparams_data['best_params']['model.patch_size']}_embed_dim-{hparams_data['best_params']['model.embed_dim']}.ckpt")
 
+    import shutil
+
+    # Define the source file and destination folder
+    source_file = 'best_model_checkpoint.txt'
+    destination_folder = 'model_storage/'
+
+    # Copy the file to the destination folder
+    shutil.copy(source_file, destination_folder)
+
+    print(f"{source_file} has been copied to {destination_folder}")
+
+
+    # Define the path to the checkpoint file and folder containing .ckpt files
+    checkpoint_file = 'best_model_checkpoint.txt'
+    checkpoint_folder = 'model_storage'
+
+    # Read the first line of the checkpoint file to get the file to keep
+    with open(checkpoint_file, 'r') as f:
+        keep_file = f.readline().strip()
+
+    # Get the full path of the file to keep
+    keep_file_path = os.path.join(checkpoint_folder, os.path.basename(keep_file))
+
+    # Iterate over files in the checkpoint folder and delete unwanted .ckpt files
+    for file in os.listdir(checkpoint_folder):
+        file_path = os.path.join(checkpoint_folder, file)
+        if file_path.endswith('.ckpt') and file_path != keep_file_path:
+            os.remove(file_path)
+            print(f"Removed: {file_path}")
+
+    print(f"Kept: {keep_file_path}")
+
+
+
 if __name__ == "__main__":
-    main()
+    main_run()

tests/test_infer.py

@@ -16,7 +16,7 @@ def config():
     with hydra.initialize(version_base=None, config_path="../configs"):
         cfg = hydra.compose(
             config_name="infer",
-            overrides=["callbacks.model_checkpoint.filename=/home/runner/work/emlo4-session-07-ajithvcoder/emlo4-session-07-ajithvcoder/model_storage/epoch-checkpoint.ckpt.ckpt"],
+            overrides=["callbacks.model_checkpoint.filename=/workspace/model_storage/epoch-checkpoint.ckpt.ckpt"],
         )
         return cfg