Refactor: Switch from singular utils.py with all fuctions to class based approach

.github/workflows/deploy.yml

@@ -32,8 +32,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install --upgrade pyyaml
-          pip install boto3
-          pip install requests
+          pip install boto3 requests schema
       - name: Deploy to ${{ matrix.environment.name }}
         continue-on-error: true
         env:
@@ -42,9 +41,10 @@ jobs:
           SPLUNK_USERNAME: ${{ secrets.SPLUNK_USERNAME }}
           SPLUNK_PASSWORD: ${{ secrets.SPLUNK_PASSWORD }}
           SPLUNK_TOKEN: ${{ secrets[matrix.environment.token_key] }}
+          DEPLOYMENT_CONFIG_PATH: "environments/${{ matrix.environment.name }}"
         run: |
           echo "Deploying to ${{ matrix.environment.name }} environment"
-          python -u deploy.py ${{ matrix.environment.name }}
+          python -u deploy.py 
       - name: Upload deployment report as artifact
         uses: actions/upload-artifact@v3
         with:

README.md

@@ -17,6 +17,10 @@ Assumptions:
 │           ├── collections.conf
 │           └── logging.conf
 ├── deploy.py
+├── modules
+│   ├── apps_processing.py
+│   ├── report_generator.py
+│   └── splunkcloud.py
 └── environments
     ├── prod
     │   ├── es
@@ -38,6 +42,7 @@ Assumptions:
   * deployment instructions per each environment (`deployment.yml`)
   * specific apps configurations (e.g. `uat/es/app1`)
 * `deploy.py` Used by the automation to perform the deployment
+* `modules/` Contains methods used in deployment automation
 
 This repository follows the same structure. Please navigate it to verify its content.
 
@@ -46,6 +51,7 @@ As mentioned, these deployment files specify the apps and configurations needed
 ```yml
 target:
   url: <deployment server URL>
+  experience: <victoria|classic>
 apps:
   # Private apps
   app1:

deploy.py

@@ -1,115 +1,97 @@
-import sys
-import json
 import os
+import boto3
 
-import yaml
+from modules.splunkcloud import SplunkCloudConnector
+from modules.apps_processing import AppFilesProcessor, DeploymentParser
+from modules.report_generator import DeploymentReportGenerator
 
-from utils import *
+AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID")
+AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")
 
-# FOR LOCAL TESTING
-# from dotenv import load_dotenv
-# load_dotenv(dotenv_path="local.env")
+DEPLOYMENT_CONFIG_PATH = os.getenv("DEPLOYMENT_CONFIG_PATH")
 
-def main():
-    if len(sys.argv) != 2:
-        print("Usage: python script.py <path_to_yaml_file>")
-        sys.exit(1)
-
-    yaml_file_path = "environments/" + sys.argv[1] + "/deployment.yml"
 
-    deployment_report = {}
+def main():
+    # Initiate deployment report
+    deployment_report = DeploymentReportGenerator()
+    # Initiate AwsS3Connector object
+    s3_connector = boto3.client("s3")
+    # Initiate DeploymentParser object
+    config = DeploymentParser()
+    # Initiate AppFilesProcessor object
+    app_processor = AppFilesProcessor(config)
+    # Initiate SplunkCloudConnector object
+    cloud_connector = SplunkCloudConnector(
+        config.url, config.cloud_experience
+    )
 
-    try:
-        data = read_yaml(yaml_file_path)
-    except FileNotFoundError:
-        print(f"Error: The file '{yaml_file_path}' was not found.")
-    except yaml.YAMLError as e:
-        print(f"Error parsing YAML file: {e}")
-        sys.exit(1)
-    ### 1. Validate data and retrieve all apps listed in deployment.yml from S3 ###
-    private_apps, splunkbase_apps = validate_data(data)
-    # List all apps in yaml file and then their S3 bucket
-    if private_apps:
-        apps = data.get("apps", {}).keys()
-        s3_buckets = [data["apps"][app]["s3-bucket"] for app in apps]
-        app_directories = [data["apps"][app]["source"] for app in apps]
-    target_url = data["target"]["url"]
-    # Download all apps from S3
-    if private_apps:
-        print("Found private apps in deployment.yml, starting deployment...")
-        for app, bucket, directory in zip(apps, s3_buckets, app_directories):
-            object_name = directory
+    # Check for private apps
+    if config.has_private_apps():
+        print("Found private apps, starting deployment...")
+        # Loop through all apps
+        for app in config.private_apps.keys():
+            bucket = config.get_bucket(app)
+            app_path = config.get_app_path(app)
             file_name = f"{app}.tgz"
             # Donwload app from S3
-            download_file_from_s3(bucket, object_name, file_name)
+            try:
+                s3_connector.download_file(bucket, app_path, file_name)
+            except Exception as e:
+                raise Exception(f"Error downloading {app_path} from {bucket}: {e}")
 
-            ### 2. Upload_local_configuration ###
-            # Check if the configuration exists for the app
-            path = os.path.join("environments", sys.argv[1], app)
-            print(path)
-            if path:
-                unpack_merge_conf_and_meta_repack(app, path)
+            ### Upload_local_configuration ###
+            # Check whether the app needs specific configs for this env
+            path = os.path.join(DEPLOYMENT_CONFIG_PATH, app)
+            if len(config.get_app_configs(app)) > 0:
+                app_processor.unpack_merge_conf_and_meta_repack(app, path)
             else:
-                print(f"No configuration found for app {app}. Skipping.")
+                print(f"No configurations needed for app {app}. Skipping.")
 
-            ### 3. Validate app for Splunk Cloud ###
-            report, token = cloud_validate_app(app)
-            if report is None:
-                print(f"App {app} failed validation.")
-                deployment_report[app] = {"validation": "failed"}
+            ### Validate app for Splunk Cloud ###
+            appinspect_handler = cloud_connector.get_appinspect_handler()
+            is_valid = appinspect_handler.validate(app)
+            if not is_valid:
+                print(f"App {app} failed validation. Skipping distribution.\n")
+                deployment_report.add_data(app, ("report", appinspect_handler.report))
+                deployment_report.add_data(app, ("validation", "failed"))
+                deployment_report.add_data(
+                    app, ("distribution", "failed due to app validation error")
+                )
                 continue
-            result = report["summary"]
-            deployment_report[app] = report
-            ### 4. If app is valid, distribute it ###
-            if (
-                result["error"] == 0
-                and result["failure"] == 0
-                and result["manual_check"] == 0
-            ):
-                distribution_status = distribute_app(app, target_url, token)
-                if distribution_status == 200:
-                    print(f"App {app} successfully distributed.\n")
-                    deployment_report[app]["distribution"] = "success"
-                else:
-                    print(f"App {app} failed distribution.")
-                    deployment_report[app][
-                        "distribution"
-                    ] = f"failed with status code: {distribution_status}"
+            ### App is valid: distribute it ###
+            deployment_report.add_data(app, ("report", appinspect_handler.report))
+            dist_succeeded, dist_status = cloud_connector.distribute(app)
+            if dist_succeeded:
+                print(f"App {app} successfully distributed.\n")
+                deployment_report.add_data(app, ("distribution", "success"))
             else:
-                print(f"App {app} failed validation. Skipping distribution.\n")
-                deployment_report[app][
-                    "distribution"
-                ] = "failed due to app validation error"
+                print(f"App {app} failed distribution.")
+                deployment_report.add_data(
+                    app,
+                    (
+                        "distribution",
+                        f"failed with status code: {dist_status}",
+                    ),
+                )
     else:
-        print("No private apps found in deployment.yml, skipping...")
+        print("No private apps found, skipping...")
 
-    ### 5. Handle Splunkbase apps ###
-    if splunkbase_apps:
-        print("Found Splunkbase apps in deployment.yml, starting deployment...")
-        splunkbase_apps_dict = data.get("splunkbase-apps", {})
-        for splunkbase_app in splunkbase_apps_dict:
-            app = splunkbase_apps_dict[splunkbase_app]
-            app_name = splunkbase_app
-            version = app['version']
-            app_id = get_app_id(app_name)
-            token = os.getenv("SPLUNK_TOKEN")
-            license = get_license_url(app_name)
-            install_status = install_splunkbase_app(app_name, app_id, version, target_url, token, license)
-            print(f"App {app_name} installation status: {install_status}")
-            deployment_report[app_name] = {
-                "splunkbase_installation": install_status,
-                "version": version,
-                "app_id": app_id,
-                }
+    ### Handle Splunkbase apps ###
+    if config.has_splunkbase_apps():
+        print("Found Splunkbase apps, starting deployment...")
+        for splunkbase_app in config.splunkbase_apps.keys():
+            version = config.get_version(app)
+            install_status = cloud_connector.install(app, version)
+            print(f"App {app} installation status: {install_status}")
+            deployment_report.add_data(
+                app,
+                {"splunkbase_installation": install_status, "version": version},
+            )
     else:
-        print("No Splunkbase apps found in deployment.yml, skipping...")
+        print("No Splunkbase apps found, skipping...")
 
-    ### 6. Save deployment report to json file ###
-    report_prefix = f"{sys.argv[1].split('/')[-2]}_{sys.argv[1].split('/')[-1]}"
-    output_dir = "artifacts"
-    os.makedirs(output_dir, exist_ok=True)
-    with open(f"{output_dir}/{report_prefix}_deployment_report.json", "w") as file:
-        json.dump(deployment_report, file)
+    ### Save deployment report to json file ###
+    deployment_report.generate_report()
 
 
 if __name__ == "__main__":

environments/prod/es/deployment.yml

@@ -1,5 +1,6 @@
 target:
   url: https://splunk-es-prod.example.com
+  experience: classic
 apps:
   app1:
     s3-bucket: splunk-apps-deployment

environments/prod/ses/deployment.yml

@@ -1,6 +1,8 @@
 target:
   url: <Staging ES URL>
+  experience: <classic|victoria>
 apps:
   app1:
     s3-bucket: splunk-apps-deployment
-    source: apps/Splunk_TA_app1.tgz
+    source: apps/Splunk_TA_app1.tgz
+splunkbase-apps:

environments/uat/es/deployment.yml

@@ -1,5 +1,6 @@
 target:
-  url: https://staging.admin.splunk.com/scv-shw-217bd09bcbf264/adminconfig/v2/apps/victoria
+  url: https://staging.admin.splunk.com/scv-shw-a7f6020a334e01
+  experience: victoria
 apps:
   Splunk_TA_app1:
     s3-bucket: splunk-apps-deployment
@@ -12,7 +13,6 @@ apps:
     config:
       - ./buttercup_app_for_splunk/*.conf
 splunkbase-apps:
-  # Splunkbase apps
   Splunk Add-on for Amazon Web Services:
     version: 7.9.0
   Cisco Networks Add-on for Splunk Enterprise:

environments/uat/ses/deployment.yml

@@ -1,13 +1,14 @@
 target:
-  url: https://staging.admin.splunk.com/scv-shw-d037e758abafa2/adminconfig/v2/apps/victoria
+  url: https://staging.admin.splunk.com/scv-shw-d037e758abafa2
+  experience: victoria
 apps:
   Splunk_TA_app1:
     s3-bucket: splunk-apps-deployment
     source: apps/Splunk_TA_app1.tgz
     config:
       - ./Splunk_TA_app1/*.conf
 splunkbase-apps:
-  TA for Tebable:
+  Tenable Add-On for Splunk:
     version: 7.0.0
-  TA for MS Azure:
+  Splunk Add on for Microsoft Azure:
     version: 4.2.0