Updated implementation (#5)

Author: nfmoore

.github/workflows/main.yaml

@@ -1,17 +1,9 @@
-name: CI-CD
-
-on:
-  push:
-    branches:
-      - development
-      - main
-  pull_request:
-    branches:
-      - development
-      - main
+name: Build-Deploy
+
+on: workflow_dispatch
 
 env:
-  PROJECT_NAME: wine-quality
+  PROJECT_NAME: employee-attrition
 
 jobs:
   build:
@@ -31,45 +23,35 @@ jobs:
         with:
           creds: ${{ secrets.AZURE_CREDENTIALS }}
 
-      - name: Setup python environment
-        uses: conda-incubator/setup-miniconda@v2
+      - uses: actions/setup-python@v2
         with:
-          activate-environment: ${{ env.PROJECT_NAME }}_env
-          environment-file: service/environment.yaml
           python-version: 3.8
-          auto-activate-base: false
+          cache: pip
+      - run: pip install -r service/requirements.txt
 
       - name: Set databricks host environment variables
         run: |
           echo "MLFLOW_TRACKING_URI=databricks" >> $GITHUB_ENV
           echo "DATABRICKS_HOST=${{ secrets.DATABRICKS_HOST }}" >> $GITHUB_ENV
           echo "DATABRICKS_TOKEN=${{ secrets.DATABRICKS_TOKEN }}" >> $GITHUB_ENV
 
-      - name: Set artifact URI environment variables
+      - name: Set artifact model environment variables
         run: |
-          echo "PREDICTION_MODEL_ARTIFACT_URI=$(jq --raw-output .prediction_model_artifact_uri service/configuration.json)" >> $GITHUB_ENV
-          echo "DRIFT_MODEL_ARTIFACT_URI=$(jq --raw-output .drift_model_artifact_uri service/configuration.json)" >> $GITHUB_ENV
-          echo "OUTLIER_MODEL_ARTIFACT_URI=$(jq --raw-output .outlier_model_artifact_uri service/configuration.json)" >> $GITHUB_ENV
+          echo "MODEL_NAME=$(jq --raw-output .model_name service/configuration.json)" >> $GITHUB_ENV
+          echo "MODEL_VERSION=$(jq --raw-output .model_version service/configuration.json)" >> $GITHUB_ENV
 
-      - name: Package model service
+      - name: Download model artifact
         run: |
-          # Download model artifacts from databricks
-          PREDICTION_MODEL_ARTIFACT_PATH=$(mlflow artifacts download -u $PREDICTION_MODEL_ARTIFACT_URI)
-          DRIFT_MODEL_ARTIFACT_PATH=$(mlflow artifacts download -u $DRIFT_MODEL_ARTIFACT_URI)
-          OUTLIER_MODEL_ARTIFACT_PATH=$(mlflow artifacts download -u $OUTLIER_MODEL_ARTIFACT_URI)
-
-          # Create directory for model package
-          mkdir -p artifacts/model_package
+          # Get model artifact uri
+          MODEL_ARTIFACT_URI=$(curl -X GET -H "Authorization: Bearer ${DATABRICKS_TOKEN}" ${DATABRICKS_HOST}/api/2.0/preview/mlflow/model-versions/get-download-uri -d '{"name": "'"${MODEL_NAME}"'", "version": "'"${MODEL_VERSION}"'"}' | jq ".artifact_uri" --raw-output)
 
-          # Package model as webservice with bentoml
-          python3 service/package_model_service.py \
-            --mlflow-prediction-model-artifiact-path $PREDICTION_MODEL_ARTIFACT_URI \
-            --mlflow-drift-model-artifiact-path $DRIFT_MODEL_ARTIFACT_URI \
-            --mlflow-outlier-model-artifiact-path $OUTLIER_MODEL_ARTIFACT_URI \
-            --package-path artifacts/model_package
+          # Download model artifacts from databricks
+          MODEL_ARTIFACT_PATH=$(mlflow artifacts download -u $MODEL_ARTIFACT_URI)
 
           # Copy model configuration to artifacts directory
+          mkdir artifacts
           cp service/configuration.json artifacts/model.json
+          cp -R $MODEL_ARTIFACT_PATH service/employee_attrition_model
 
       - name: ACR login
         uses: azure/docker-login@v1
@@ -80,7 +62,7 @@ jobs:
 
       - name: Build and push image
         run: |
-          docker build -t ${{ env.PROJECT_NAME }} artifacts/model_package
+          docker build -t ${{ env.PROJECT_NAME }} service
           docker tag ${{ env.PROJECT_NAME }} ${{ secrets.CONTAINER_REGISTRY_NAME }}.azurecr.io/${{ env.PROJECT_NAME }}:${{ github.sha }}
           docker push ${{ secrets.CONTAINER_REGISTRY_NAME }}.azurecr.io/${{ env.PROJECT_NAME }}:${{ github.sha }}
 
@@ -142,14 +124,14 @@ jobs:
             ${{ secrets.CONTAINER_REGISTRY_IMAGE_PULL_SECRET }}
           namespace: default
 
+      - name: Set artifact model environment variables
+        run: |
+          echo "MODEL_NAME=$(jq --raw-output .model_name model.json)" >> $GITHUB_ENV
+          echo "MODEL_VERSION=$(jq --raw-output .model_version model.json)" >> $GITHUB_ENV
+
       - name: Transition models to staging
         run: |
-          for i in {$PREDICTION_MODEL_ARTIFACT_URI,$DRIFT_MODEL_ARTIFACT_URI,$OUTLIER_MODEL_ARTIFACT_URI}; 
-          do 
-            MODEL_NAME=$(echo $i | cut -c 9- | cut -d '/' -f1) ;
-            MODEL_VERSION=$(echo $i | cut -c 9- | cut -d '/' -f2) ;
-            curl -i -X POST -H "Authorization: Bearer ${DATABRICKS_TOKEN}" ${DATABRICKS_HOST}/api/2.0/preview/mlflow/model-versions/transition-stage -d '{"name": "'"${MODEL_NAME}"'", "version": "'"${MODEL_VERSION}"'", "stage": "Staging", "archive_existing_versions": true}' ;
-          done
+          curl -i -X POST -H "Authorization: Bearer ${{ secrets.DATABRICKS_TOKEN }}" ${{ secrets.DATABRICKS_HOST }}/api/2.0/preview/mlflow/model-versions/transition-stage -d '{"name": "'"${MODEL_NAME}"'", "version": "'"${MODEL_VERSION}"'", "stage": "Staging", "archive_existing_versions": true}' ;
 
   production:
     name: Production
@@ -197,11 +179,11 @@ jobs:
             ${{ secrets.CONTAINER_REGISTRY_IMAGE_PULL_SECRET }}
           namespace: default
 
-      - name: Transition models to production
+      - name: Set artifact model environment variables
+        run: |
+          echo "MODEL_NAME=$(jq --raw-output .model_name model.json)" >> $GITHUB_ENV
+          echo "MODEL_VERSION=$(jq --raw-output .model_version model.json)" >> $GITHUB_ENV
+
+      - name: Transition models to staging
         run: |
-          for i in {$PREDICTION_MODEL_ARTIFACT_URI,$DRIFT_MODEL_ARTIFACT_URI,$OUTLIER_MODEL_ARTIFACT_URI}; 
-          do 
-            MODEL_NAME=$(echo $i | cut -c 9- | cut -d '/' -f1) ;
-            MODEL_VERSION=$(echo $i | cut -c 9- | cut -d '/' -f2) ;
-            curl -i -X POST -H "Authorization: Bearer ${DATABRICKS_TOKEN}" ${DATABRICKS_HOST}/api/2.0/preview/mlflow/model-versions/transition-stage -d '{"name": "'"${MODEL_NAME}"'", "version": "'"${MODEL_VERSION}"'", "stage": "Production", "archive_existing_versions": true}' ;
-          done
+          curl -i -X POST -H "Authorization: Bearer ${{ secrets.DATABRICKS_TOKEN }}" ${{ secrets.DATABRICKS_HOST }}/api/2.0/preview/mlflow/model-versions/transition-stage -d '{"name": "'"${MODEL_NAME}"'", "version": "'"${MODEL_VERSION}"'", "stage": "Production", "archive_existing_versions": true}' ;

.gitignore

@@ -129,5 +129,5 @@ dmypy.json
 .pyre/
 
 # Custom
-model_package
+employee_attrition_model
 .DS_Store

infrastructure/main.bicep

@@ -0,0 +1,28 @@
+//********************************************************
+// Modules
+//********************************************************
+
+module m_databricks './modules/databricks.bicep' = {
+  name: 'm_databricks'
+  params: {
+    resourceInstance: '01'
+  }
+}
+
+module m_microservices_01 './modules/microservices.bicep' = {
+  name: 'm_microservices_01'
+  params: {
+    resourceInstance: '01'
+  }
+}
+
+module m_microservices_02 './modules/microservices.bicep' = {
+  name: 'm_microservices_02'
+  params: {
+    resourceInstance: '02'
+    useExistingContainerRegistry: true
+    useExistingLogAnalyticsWorkspace: true
+    containerRegistryName: m_microservices_01.outputs.containerRegistryName
+    logAnalyticsWorkspaceName: m_microservices_01.outputs.logAnalyticsWorkspaceName
+  }
+}