From 62be3947111e8addf1da7a1fe36e4870dad40920 Mon Sep 17 00:00:00 2001 From: bdebek-splunk Date: Thu, 16 Jan 2025 11:53:49 +0100 Subject: [PATCH 01/20] refactor: swithc from singular utils.py to app_processing, aws_3 and splunkcloud modules with class based interfaces --- deploy.py | 58 +++-- modules/apps_processing.py | 209 ++++++++++++++++ modules/aws_s3.py | 24 ++ modules/splunkcloud.py | 258 ++++++++++++++++++++ utils.py | 471 ------------------------------------- 5 files changed, 527 insertions(+), 493 deletions(-) create mode 100644 modules/apps_processing.py create mode 100644 modules/aws_s3.py create mode 100644 modules/splunkcloud.py delete mode 100644 utils.py diff --git a/deploy.py b/deploy.py index 3cf0329..1abcb78 100644 --- a/deploy.py +++ b/deploy.py @@ -2,14 +2,22 @@ import json import os -import yaml - -from utils import * +from modules.splunkcloud import SplunkCloudConnector +from modules.aws_s3 import AwsS3Connector +from modules.apps_processing import AppFilesProcessor # FOR LOCAL TESTING # from dotenv import load_dotenv # load_dotenv(dotenv_path="local.env") +SPLUNK_USERNAME = os.getenv("SPLUNK_USERNAME") +SPLUNK_PASSWORD = os.getenv("SPLUNK_PASSWORD") +SPLUNK_TOKEN = os.getenv("SPLUNK_TOKEN") + +AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") +AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") + + def main(): if len(sys.argv) != 2: print("Usage: python script.py ") @@ -17,43 +25,48 @@ def main(): yaml_file_path = "environments/" + sys.argv[1] + "/deployment.yml" + # Initiate deployment report deployment_report = {} - try: - data = read_yaml(yaml_file_path) - except FileNotFoundError: - print(f"Error: The file '{yaml_file_path}' was not found.") - except yaml.YAMLError as e: - print(f"Error parsing YAML file: {e}") - sys.exit(1) + # Initiate AppFilesProcessor object + app_processor = AppFilesProcessor(yaml_file_path) + ### 1. Validate data and retrieve all apps listed in deployment.yml from S3 ### - private_apps, splunkbase_apps = validate_data(data) + data, private_apps, splunkbase_apps = app_processor.validate_data() # List all apps in yaml file and then their S3 bucket if private_apps: apps = data.get("apps", {}).keys() s3_buckets = [data["apps"][app]["s3-bucket"] for app in apps] app_directories = [data["apps"][app]["source"] for app in apps] target_url = data["target"]["url"] - # Download all apps from S3 + + # Initiate AwsS3Connector object + s3_connector = AwsS3Connector(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) + # Check for private apps if private_apps: print("Found private apps in deployment.yml, starting deployment...") + # Loop through all apps for app, bucket, directory in zip(apps, s3_buckets, app_directories): object_name = directory file_name = f"{app}.tgz" # Donwload app from S3 - download_file_from_s3(bucket, object_name, file_name) + s3_connector.download_file_from_s3(bucket, object_name, file_name) ### 2. Upload_local_configuration ### # Check if the configuration exists for the app path = os.path.join("environments", sys.argv[1], app) print(path) if path: - unpack_merge_conf_and_meta_repack(app, path) + app_processor.unpack_merge_conf_and_meta_repack(app, path) else: print(f"No configuration found for app {app}. Skipping.") ### 3. Validate app for Splunk Cloud ### - report, token = cloud_validate_app(app) + # Initiate SplunkCloudConnector object + cloud_connector = SplunkCloudConnector( + SPLUNK_USERNAME, SPLUNK_PASSWORD, SPLUNK_TOKEN, target_url + ) + report, token = cloud_connector.cloud_validate_app(app) if report is None: print(f"App {app} failed validation.") deployment_report[app] = {"validation": "failed"} @@ -66,7 +79,7 @@ def main(): and result["failure"] == 0 and result["manual_check"] == 0 ): - distribution_status = distribute_app(app, target_url, token) + distribution_status = cloud_connector.distribute_app(app, token) if distribution_status == 200: print(f"App {app} successfully distributed.\n") deployment_report[app]["distribution"] = "success" @@ -90,17 +103,18 @@ def main(): for splunkbase_app in splunkbase_apps_dict: app = splunkbase_apps_dict[splunkbase_app] app_name = splunkbase_app - version = app['version'] - app_id = get_app_id(app_name) - token = os.getenv("SPLUNK_TOKEN") - license = get_license_url(app_name) - install_status = install_splunkbase_app(app_name, app_id, version, target_url, token, license) + version = app["version"] + app_id = cloud_connector.get_app_id(app_name) + license = cloud_connector.get_license_url(app_name) + install_status = cloud_connector.install_splunkbase_app( + app_name, app_id, version, license + ) print(f"App {app_name} installation status: {install_status}") deployment_report[app_name] = { "splunkbase_installation": install_status, "version": version, "app_id": app_id, - } + } else: print("No Splunkbase apps found in deployment.yml, skipping...") diff --git a/modules/apps_processing.py b/modules/apps_processing.py new file mode 100644 index 0000000..2dde9bb --- /dev/null +++ b/modules/apps_processing.py @@ -0,0 +1,209 @@ +import sys +import os +import yaml +import shutil +import configparser +import tarfile +from io import StringIO + + +class AppFilesProcessor: + """Class for handling local app files and configurations.""" + + def __init__(self, yml_path): + self.yml_path = yml_path + + def _read_yaml(self) -> dict: + """Read and return the contents of a YAML file.""" + file_path = self.yml_path + with open(file_path, "r") as file: + return yaml.safe_load(file) + + def validate_data(self) -> tuple: + """ + Validate the data in the YAML file. + + Return boolean values for private_apps and splunkbase_apps presence in the environment configuration + + validate_data(data) -> (bool, bool) + """ + try: + data = self._read_yaml() + except FileNotFoundError: + print(f"Error: The file '{self.yml_path}' was not found.") + except yaml.YAMLError as e: + print(f"Error parsing YAML file: {e}") + sys.exit(1) + + if "apps" not in data: + print("Error: The 'apps' key is missing in deploy.yml fime.") + sys.exit(1) + if "target" not in data: + print("Error: The 'target' key is missing in deploy.yml file.") + sys.exit(1) + if "url" not in data["target"]: + print("Error: The 'url' key is missing in the 'target' section.") + sys.exit(1) + if "splunkbase-apps" not in data: + print("Error: The 'splunkbase-apps' key is missing.") + sys.exit(1) + + app_dict = data.get("apps", {}) + splunkbase_dict = data.get("splunkbase-apps", {}) + + private_apps = True if app_dict else False + splunkbase_apps = True if splunkbase_dict else False + + return data, private_apps, splunkbase_apps + + def _preprocess_empty_headers(self, file_path: str) -> list: + """ + Preprocess the file to handle empty section headers by replacing `[]` with a valid section name. + """ + valid_lines = [] + with open(file_path, "r") as file: + for line in file: + # Replace empty section headers with a placeholder + if line.strip() == "[]": + valid_lines.append("[DEFAULT]\n") # Or any placeholder section name + else: + valid_lines.append(line) + return valid_lines + + def _replace_default_with_empty_header(self, file_path: str) -> None: + """ + Replace '[DEFAULT]' header with '[]' in the specified file. + """ + with open(file_path, "r") as file: + lines = file.readlines() + + with open(file_path, "w") as file: + for line in lines: + # Replace '[DEFAULT]' with '[]' + if line.strip() == "[DEFAULT]": + file.write("[]\n") + else: + file.write(line) + + def merge_or_copy_conf(self, source_path: str, dest_path: str) -> None: + # Get the filename from the source path + filename = os.path.basename(source_path) + dest_file = os.path.join(dest_path, filename) + + # Check if the file exists in the destination directory + if not os.path.exists(dest_file): + # If the file doesn't exist, copy it + shutil.copy(source_path, dest_path) + print(f"Copied {filename} to {dest_path}") + else: + # If the file exists, merge the configurations + print(f"Merging {filename} with existing file in {dest_path}") + + # Read the source file + source_config = configparser.ConfigParser() + source_config.read(source_path) + + # Read the destination file + dest_config = configparser.ConfigParser() + dest_config.read(dest_file) + + # Merge source into destination + for section in source_config.sections(): + if not dest_config.has_section(section): + dest_config.add_section(section) + for option, value in source_config.items(section): + dest_config.set(section, option, value) + + # Write the merged configuration back to the destination file + with open(dest_file, "w") as file: + dest_config.write(file) + print(f"Merged configuration saved to {dest_file}") + + def merge_or_copy_meta(self, local_meta_file: str, default_dir: str) -> None: + """Merge local.meta with default.meta""" + filename = os.path.basename(local_meta_file) + dest_file = os.path.join(default_dir, "default.meta") + + # Check if the file exists in the destination directory + if not os.path.exists(dest_file): + # If the file doesn't exist, copy it + shutil.copy(local_meta_file, dest_file) + print(f"Copied {filename} to {dest_file}") + else: + # If the file exists, merge the configurations + print(f"Merging {filename} with existing file in {dest_file}") + + # Preprocess the default file + default_preprocessed_lines = self._preprocess_empty_headers(dest_file) + default_preprocessed_content = StringIO("".join(default_preprocessed_lines)) + + # Read the default.meta file + default_meta = configparser.ConfigParser() + default_meta.read_file(default_preprocessed_content) + + # Preprocess the local file + local_preprocessed_lines = self._preprocess_empty_headers(local_meta_file) + local_preprocessed_content = StringIO("".join(local_preprocessed_lines)) + + # Read the local.meta file + local_meta = configparser.ConfigParser() + local_meta.read_file(local_preprocessed_content) + + # Merge local.meta into default.meta + for section in local_meta.sections(): + if not default_meta.has_section(section): + default_meta.add_section(section) + for option, value in local_meta.items(section): + if default_meta.has_option(section, option): + # Merge logic: Option exists in both, decide whether to overwrite + default_value = default_meta.get(section, option) + if value != default_value: + print( + f"Conflict detected: {section} {option} - {default_value} -> {value}" + ) + # Overwrite the option in default.meta + default_meta.set(section, option, value) + default_meta.set(section, option, value) + + # Write the merged configuration back to the output file + with open(dest_file, "w") as file: + default_meta.write(file) + + # Replace '[DEFAULT]' with '[]' in the output file + self._replace_default_with_empty_header(dest_file) + + print(f"Merged metadata saved to {dest_file}") + + def unpack_merge_conf_and_meta_repack(self, app: str, path: str) -> None: + """Unpack the app, load environment configuration files and repack the app.""" + temp_dir = "temp_unpack" + os.makedirs(temp_dir, exist_ok=True) + + # Unpack the tar.gz file + with tarfile.open(f"{app}.tgz", "r:gz") as tar: + tar.extractall(path=temp_dir) + # Create default directory for unpacked app + base_default_dir = f"{temp_dir}/{app}" + # Load the environment configuration files + app_dir = path + # Copy all .conf files in app_dir to temp_dir of unpacked app + for file in os.listdir(app_dir): + if file.endswith(".conf"): + default_dir = base_default_dir + "/default" + os.makedirs(default_dir, exist_ok=True) + source_path = os.path.join(app_dir, file) + self.merge_or_copy_conf(source_path, default_dir) + # Copy all metadata files in app_dir to temp_dir of unpacked app + for file in os.listdir(app_dir): + if file.endswith(".meta"): + default_dir = base_default_dir + "/metadata" + os.makedirs(default_dir, exist_ok=True) + source_path = os.path.join(app_dir, file) + self.merge_or_copy_meta(source_path, default_dir) + # Repack the app and place it in the root directory + with tarfile.open(f"{app}.tgz", "w:gz") as tar: + for root, _, files in os.walk(f"{temp_dir}/{app}"): + for file in files: + full_path = os.path.join(root, file) + arcname = os.path.relpath(full_path, temp_dir) + tar.add(full_path, arcname=arcname) diff --git a/modules/aws_s3.py b/modules/aws_s3.py new file mode 100644 index 0000000..78b4faa --- /dev/null +++ b/modules/aws_s3.py @@ -0,0 +1,24 @@ +import boto3 + + +class AwsS3Connector: + """Class to connect to AWS S3 and download files.""" + + def __init__(self, aws_access_key_id, aws_secret_access_key): + self.aws_access_key_id = aws_access_key_id + self.aws_secret_access_key = aws_secret_access_key + + def download_file_from_s3( + self, bucket_name: str, object_name: str, file_name: str + ) -> None: + """Download a file from an S3 bucket.""" + s3 = boto3.client( + "s3", + aws_access_key_id=self.aws_access_key_id, + aws_secret_access_key=self.aws_secret_access_key, + ) + try: + s3.download_file(bucket_name, object_name, file_name) + print(f"Downloaded {object_name} from {bucket_name} to {file_name}") + except Exception as e: + print(f"Error downloading {object_name} from {bucket_name}: {e}") diff --git a/modules/splunkcloud.py b/modules/splunkcloud.py new file mode 100644 index 0000000..0153ced --- /dev/null +++ b/modules/splunkcloud.py @@ -0,0 +1,258 @@ +import os +import requests +import time + +import xml.etree.ElementTree as ET + + +class SplunkCloudConnector: + """Class for connecting to Splunk Cloud and Splunkbase.""" + + SPLUNK_AUTH_BASE_URL = "https://api.splunk.com/2.0/rest/login/splunk" + SPLUNK_APPINSPECT_BASE_URL = "https://appinspect.splunk.com/v1" + SPLUNKBASE_BASE_URL = "https://splunkbase.splunk.com/api/account:login" + SPLUNKBASE_APP_URL = "https://splunkbase.splunk.com/api/v1/app" + + def __init__( + self, + splunk_username: str = None, + splunk_password: str = None, + splunk_token: str = None, + splunk_host: str = None, + ): + self.splunk_username = splunk_username + self.splunk_password = splunk_password + self.splunk_token = splunk_token + self.splunk_host = splunk_host + + def get_appinspect_token(self) -> str: + """ + Authenticate to the Splunk Cloud. + + get_appinspect_token() -> token : str + """ + url = self.SPLUNK_AUTH_BASE_URL + username = self.splunk_username + password = self.splunk_password + + response = requests.get(url, auth=(username, password)) + token = response.json()["data"]["token"] + return token + + def validation_request_helper(self, url: str, headers: dict, files: dict) -> str: + """ + Helper function to make a validation request and return the request ID. + + validation_request_helper(url, headers, files) -> request_id : str + """ + try: + response = requests.post(url, headers=headers, files=files, timeout=120) + response_json = response.json() + request_id = response_json["request_id"] + except requests.exceptions.RequestException as e: + print(f"Error making app validation request: {e}") + return None + return request_id + + def cloud_validate_app(self, app: str) -> tuple: + """ + Validate the app for the Splunk Cloud. + + cloud_validate_app(app) -> report : dict, token : str + """ + token = self.get_appinspect_token() + base_url = self.SPLUNK_APPINSPECT_BASE_URL + url = f"{base_url}/app/validate" + + headers = {"Authorization": f"Bearer {token}"} + app_file_path = f"{app}.tgz" + + print(f"Validating app {app}...") + with open(app_file_path, "rb") as file: + files = {"app_package": file} + request_id = self.validation_request_helper(url, headers, files) + headers = {"Authorization": f"Bearer {token}"} + status_url = f"{base_url}/app/validate/status/{request_id}?included_tags=private_victoria" + try: + response_status = requests.get(status_url, headers=headers) + except requests.exceptions.RequestException as e: + print(f"Error: {e}") + return None, None + + max_retries = 60 # Maximum number of retries + retries = 0 + response_status_json = response_status.json() + + while response_status_json["status"] != "SUCCESS" and retries < max_retries: + response_status = requests.get(status_url, headers=headers) + response_status_json = response_status.json() + retries += 1 + if response_status_json["status"] == "FAILURE": + print( + f"App {app} failed validation: {response_status_json['errors']}" + ) + break + else: + print(f"App {app} awaiting validation...") + print(f"Current status: {response_status_json['status']}") + time.sleep(10) + response_status = requests.get(status_url, headers=headers) + response_status_json = response_status.json() + continue + if retries == max_retries: + print(f"App {app} validation timed out.") + return + + print(f"Current status: {response_status_json['status']}") + if response_status_json["status"] == "SUCCESS": + print("App validation successful.") + print("Installing app...") + + response_report = requests.get( + f"{base_url}/app/report/{request_id}?included_tags=private_victoria", + headers=headers, + ) + report = response_report.json() + result = report["summary"] + print(result) + + return report, token + + def distribute_app(self, app: str, token: str) -> int: + """ + Distribute the app to the target URL. + + distribute_app(app, target_url, token) -> status_code : int + """ + print(f"Distributing {app} to {self.splunk_host}") + target_url = self.splunk_host + url = target_url + admin_token = self.splunk_token + headers = { + "X-Splunk-Authorization": token, + "Authorization": f"Bearer {admin_token}", + "ACS-Legal-Ack": "Y", + } + file_path = f"{app}.tgz" + try: + with open(file_path, "rb") as file: + response = requests.post(url, headers=headers, data=file) + print( + f"Distributed {app} to {target_url} with response: {response.status_code} {response.text}" + ) + except Exception as e: + print(f"Error distributing {app} to {self.splunk_host}: {e}") + return 500 + + return response.status_code + + def authenticate_splunkbase(self) -> str: + """ + Authenticate to Splunkbase. + + authenticate_splunkbase() -> token : str + """ + url = self.SPLUNKBASE_BASE_URL + data = {"username": self.splunk_username, "password": self.splunk_password} + response = requests.post(url, data=data) + + if response.ok: + # Parse the XML response + xml_root = ET.fromstring(response.text) + # Extract the token from the tag + namespace = {"atom": "http://www.w3.org/2005/Atom"} # Define the namespace + splunkbase_token = xml_root.find( + "atom:id", namespace + ).text # Find the tag with the namespace + return splunkbase_token + else: + print("Splunkbase login failed!") + print(f"Status code: {response.status_code}") + print(response.text) + return None + + def install_splunkbase_app( + self, app: str, app_id: str, version: str, licence: str + ) -> str: + """ + Install a Splunkbase app. + + install_splunkbase_app(app, app_id, version, target_url, token, licence) -> status : str + """ + # Authenticate to Splunkbase + splunkbase_token = self.authenticate_splunkbase() + # Install the app + target_url = self.splunk_host + token = self.splunk_token + + url = f"{target_url}?splunkbase=true" + + headers = { + "X-Splunkbase-Authorization": splunkbase_token, + "ACS-Licensing-Ack": licence, + "Authorization": f"Bearer {token}", + "Content-Type": "application/x-www-form-urlencoded", + } + data = {"splunkbaseID": app_id, "version": version} + + response = requests.post(url, headers=headers, data=data) + # Handle the case where the app is already installed + if response.status_code == 409: + print(f"App {app} is already installed.") + print(f"Updating app {app} to version {version}...") + # Get app name + url = f"https://splunkbase.splunk.com/api/v1/app/{app_id}" + response = requests.get(url) + app_name = response.json().get("appid") + print(f"App name: {app_name}") + # Update the app + url = f"{target_url}/{app_name}" + data = {"version": version} + response = requests.patch(url, headers=headers, data=data) + return "success - existing app updated" + elif response.ok: + request_status = response.json()["status"] + print(f"Request status: {request_status}") + if request_status in ("installed", "processing"): + print(f"App {app} version {version} installation successful.") + return "success" + else: + print(f"App {app} version {version} installation failed.") + return f"failed with status: {request_status} - {response.text}" + else: + print("Request failed!") + print(f"Status code: {response.status_code}") + print(response.text) + return f"failed with status code: {response.status_code} - {response.text}" + + def get_app_id(self, app_name: str) -> str: + """ + Get the Splunkbase app ID. + + get_app_id(app_name) -> app_id : str + """ + url = self.SPLUNKBASE_APP_URL + params = {"query": app_name, "limit": 1} + response = requests.get(url, params=params) + if len(response.json().get("results")) > 0: + app_id = response.json().get("results")[0].get("uid") + return app_id + else: + print(f"App {app_name} not found on Splunkbase.") + return None + + def get_license_url(self, app_name: str) -> str: + """ + Get the licence URL for a Splunkbase app. + + get_licence_url(app_name) -> licence_url : str + """ + url = self.SPLUNKBASE_APP_URL + params = {"query": app_name, "limit": 1} + response = requests.get(url, params=params) + if len(response.json().get("results")) > 0: + license_url = response.json().get("results")[0].get("license_url") + return license_url + else: + print(f"App {app_name} not found on Splunkbase.") + return None diff --git a/utils.py b/utils.py deleted file mode 100644 index f32d2a9..0000000 --- a/utils.py +++ /dev/null @@ -1,471 +0,0 @@ -import sys -import time -import json -import os -import tarfile -import shutil - -import yaml -import boto3 -import requests -import itertools -import configparser -import xml.etree.ElementTree as ET - -from io import StringIO - - -SPLUNK_APPINSPECT_BASE_URL = "https://appinspect.splunk.com/v1" -SPLUNKBASE_BASE_URL = "https://splunkbase.splunk.com/api/account:login" -SPLUNK_AUTH_BASE_URL = "https://api.splunk.com/2.0/rest/login/splunk" - -def read_yaml(file_path: str) -> dict: - """Read and return the contents of a YAML file.""" - with open(file_path, "r") as file: - return yaml.safe_load(file) - -def check_all_letter_cases(base_path: str, app_name: str) -> str: - """Check all letter cases for the app configuration.""" - # Generate all case combinations of "app" - case_variations = map("".join, itertools.product(*([char.lower(), char.upper()] for char in app_name))) - - # Check each variation in the path - for variation in case_variations: - path = os.path.join("environments", base_path, variation) - if os.path.exists(path): - print(f"Found: {path}") - return path - return None - -def validate_data(data: dict) -> tuple: - """ - Validate the data in the YAML file. - - Return boolean values for private_apps and splunkbase_apps presence in the environment configuration - - validate_data(data) -> (bool, bool) - """ - if "apps" not in data: - print("Error: The 'apps' key is missing in deploy.yml fime.") - sys.exit(1) - if "target" not in data: - print("Error: The 'target' key is missing in deploy.yml file.") - sys.exit(1) - if "url" not in data["target"]: - print("Error: The 'url' key is missing in the 'target' section.") - sys.exit(1) - if "splunkbase-apps" not in data: - print("Error: The 'splunkbase-apps' key is missing.") - sys.exit(1) - - app_dict = data.get("apps", {}) - splunkbase_dict = data.get("splunkbase-apps", {}) - - private_apps = True if app_dict else False - splunkbase_apps = True if splunkbase_dict else False - - return private_apps, splunkbase_apps - -def download_file_from_s3(bucket_name: str, object_name: str, file_name: str) -> None: - """Download a file from an S3 bucket.""" - s3 = boto3.client( - "s3", - aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), - aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), - ) - try: - s3.download_file(bucket_name, object_name, file_name) - print(f"Downloaded {object_name} from {bucket_name} to {file_name}") - except Exception as e: - print(f"Error downloading {object_name} from {bucket_name}: {e}") - -def preprocess_empty_headers(file_path: str) -> list: - """ - Preprocess the file to handle empty section headers by replacing `[]` with a valid section name. - """ - valid_lines = [] - with open(file_path, 'r') as file: - for line in file: - # Replace empty section headers with a placeholder - if line.strip() == "[]": - valid_lines.append("[DEFAULT]\n") # Or any placeholder section name - else: - valid_lines.append(line) - return valid_lines - -def replace_default_with_empty_header(file_path: str) -> None: - """ - Replace '[DEFAULT]' header with '[]' in the specified file. - """ - with open(file_path, 'r') as file: - lines = file.readlines() - - with open(file_path, 'w') as file: - for line in lines: - # Replace '[DEFAULT]' with '[]' - if line.strip() == "[DEFAULT]": - file.write("[]\n") - else: - file.write(line) - -def merge_or_copy_conf(source_path: str, dest_path: str) -> None: - # Get the filename from the source path - filename = os.path.basename(source_path) - dest_file = os.path.join(dest_path, filename) - - # Check if the file exists in the destination directory - if not os.path.exists(dest_file): - # If the file doesn't exist, copy it - shutil.copy(source_path, dest_path) - print(f"Copied {filename} to {dest_path}") - else: - # If the file exists, merge the configurations - print(f"Merging {filename} with existing file in {dest_path}") - - # Read the source file - source_config = configparser.ConfigParser() - source_config.read(source_path) - - # Read the destination file - dest_config = configparser.ConfigParser() - dest_config.read(dest_file) - - # Merge source into destination - for section in source_config.sections(): - if not dest_config.has_section(section): - dest_config.add_section(section) - for option, value in source_config.items(section): - dest_config.set(section, option, value) - - # Write the merged configuration back to the destination file - with open(dest_file, 'w') as file: - dest_config.write(file) - print(f"Merged configuration saved to {dest_file}") - -def merge_or_copy_meta(local_meta_file: str, default_dir: str) -> None: - """Merge local.meta with default.meta""" - filename = os.path.basename(local_meta_file) - dest_file = os.path.join(default_dir, "default.meta") - - # Check if the file exists in the destination directory - if not os.path.exists(dest_file): - # If the file doesn't exist, copy it - shutil.copy(local_meta_file, dest_file) - print(f"Copied {filename} to {dest_file}") - else: - # If the file exists, merge the configurations - print(f"Merging {filename} with existing file in {dest_file}") - - # Preprocess the default file - default_preprocessed_lines = preprocess_empty_headers(dest_file) - default_preprocessed_content = StringIO(''.join(default_preprocessed_lines)) - - # Read the default.meta file - default_meta = configparser.ConfigParser() - default_meta.read_file(default_preprocessed_content) - - # Preprocess the local file - local_preprocessed_lines = preprocess_empty_headers(local_meta_file) - local_preprocessed_content = StringIO(''.join(local_preprocessed_lines)) - - # Read the local.meta file - local_meta = configparser.ConfigParser() - local_meta.read_file(local_preprocessed_content) - - # Merge local.meta into default.meta - for section in local_meta.sections(): - if not default_meta.has_section(section): - default_meta.add_section(section) - for option, value in local_meta.items(section): - if default_meta.has_option(section, option): - # Merge logic: Option exists in both, decide whether to overwrite - default_value = default_meta.get(section, option) - if value != default_value: - print(f"Conflict detected: {section} {option} - {default_value} -> {value}") - # Overwrite the option in default.meta - default_meta.set(section, option, value) - default_meta.set(section, option, value) - - # Write the merged configuration back to the output file - with open(dest_file, 'w') as file: - default_meta.write(file) - - # Replace '[DEFAULT]' with '[]' in the output file - replace_default_with_empty_header(dest_file) - - print(f"Merged metadata saved to {dest_file}") - - -def unpack_merge_conf_and_meta_repack(app: str, path: str) -> None: - """Unpack the app, load environment configuration files and repack the app.""" - temp_dir = "temp_unpack" - os.makedirs(temp_dir, exist_ok=True) - - # Unpack the tar.gz file - with tarfile.open(f"{app}.tgz", "r:gz") as tar: - tar.extractall(path=temp_dir) - # Create default directory for unpacked app - base_default_dir = f"{temp_dir}/{app}" - # Load the environment configuration files - app_dir = path - # Copy all .conf files in app_dir to temp_dir of unpacked app - for file in os.listdir(app_dir): - if file.endswith(".conf"): - default_dir = base_default_dir + "/default" - os.makedirs(default_dir, exist_ok=True) - source_path = os.path.join(app_dir, file) - merge_or_copy_conf(source_path, default_dir) - # Copy all metadata files in app_dir to temp_dir of unpacked app - for file in os.listdir(app_dir): - if file.endswith(".meta"): - default_dir = base_default_dir + "/metadata" - os.makedirs(default_dir, exist_ok=True) - source_path = os.path.join(app_dir, file) - merge_or_copy_meta(source_path, default_dir) - # Repack the app and place it in the root directory - with tarfile.open(f"{app}.tgz", "w:gz") as tar: - for root, _, files in os.walk(f"{temp_dir}/{app}"): - for file in files: - full_path = os.path.join(root, file) - arcname = os.path.relpath(full_path, temp_dir) - tar.add(full_path, arcname=arcname) - -def get_appinspect_token() -> str: - """ - Authenticate to the Splunk Cloud. - - get_appinspect_token() -> token : str - """ - url = SPLUNK_AUTH_BASE_URL - username = os.getenv("SPLUNK_USERNAME") - password = os.getenv("SPLUNK_PASSWORD") - - response = requests.get(url, auth=(username, password)) - token = response.json()["data"]["token"] - return token - - -def validation_request_helper(url: str, headers: dict , files: dict) -> str: - """ - Helper function to make a validation request and return the request ID. - - validation_request_helper(url, headers, files) -> request_id : str - """ - try: - response = requests.post(url, headers=headers, files=files, timeout=120) - response_json = response.json() - request_id = response_json["request_id"] - except requests.exceptions.RequestException as e: - print(f"Error: {e}") - return None - return request_id - - -def cloud_validate_app(app: str) -> tuple: - """ - Validate the app for the Splunk Cloud. - - cloud_validate_app(app) -> report : dict, token : str - """ - token = get_appinspect_token() - base_url = SPLUNK_APPINSPECT_BASE_URL - url = f"{base_url}/app/validate" - - headers = {"Authorization": f"Bearer {token}"} - app_file_path = f"{app}.tgz" - - print(f"Validating app {app}...") - with open(app_file_path, "rb") as file: - files = {"app_package": file} - request_id = validation_request_helper(url, headers, files) - headers = {"Authorization": f"Bearer {token}"} - status_url = f"{base_url}/app/validate/status/{request_id}?included_tags=private_victoria" - try: - response_status = requests.get(status_url, headers=headers) - except requests.exceptions.RequestException as e: - print(f"Error: {e}") - return None, None - - max_retries = 60 # Maximum number of retries - retries = 0 - response_status_json = response_status.json() - - while response_status_json["status"] != "SUCCESS" and retries < max_retries: - response_status = requests.get(status_url, headers=headers) - response_status_json = response_status.json() - retries += 1 - if response_status_json["status"] == "FAILURE": - print(f"App {app} failed validation: {response_status_json['errors']}") - break - else: - print(f"App {app} awaiting validation...") - print(f"Current status: {response_status_json['status']}") - time.sleep(10) - response_status = requests.get(status_url, headers=headers) - response_status_json = response_status.json() - continue - if retries == max_retries: - print(f"App {app} validation timed out.") - return - - print(f"Current status: {response_status_json['status']}") - if response_status_json["status"] == "SUCCESS": - print("App validation successful.") - print("Installing app...") - - response_report = requests.get( - f"{base_url}/app/report/{request_id}?included_tags=private_victoria", - headers=headers, - ) - report = response_report.json() - result = report["summary"] - print(result) - - return report, token - - -def distribute_app(app: str, target_url: str, token: str) -> int: - """ - Distribute the app to the target URL. - - distribute_app(app, target_url, token) -> status_code : int - """ - print(f"Distributing {app} to {target_url}") - url = target_url - admin_token = os.getenv("SPLUNK_TOKEN") - headers = { - "X-Splunk-Authorization": token, - "Authorization": f"Bearer {admin_token}", - "ACS-Legal-Ack": "Y", - } - file_path = f"{app}.tgz" - try: - with open(file_path, "rb") as file: - response = requests.post(url, headers=headers, data=file) - print( - f"Distributed {app} to {target_url} with response: {response.status_code} {response.text}" - ) - except Exception as e: - print(f"Error distributing {app} to {target_url}: {e}") - return 500 - - return response.status_code - -def authenticate_splunkbase() -> str: - """ - Authenticate to Splunkbase. - - authenticate_splunkbase() -> token : str - """ - url = SPLUNKBASE_BASE_URL - data = { - 'username': os.getenv("SPLUNK_USERNAME"), - 'password': os.getenv("SPLUNK_PASSWORD") - } - response = requests.post(url, data=data) - - if response.ok: - # Parse the XML response - xml_root = ET.fromstring(response.text) - # Extract the token from the tag - namespace = {'atom': 'http://www.w3.org/2005/Atom'} # Define the namespace - splunkbase_token = xml_root.find('atom:id', namespace).text # Find the tag with the namespace - return splunkbase_token - else: - print("Splunkbase login failed!") - print(f"Status code: {response.status_code}") - print(response.text) - return None - -def install_splunkbase_app(app: str, app_id: str, version: str, target_url: str, token: str, licence: str) -> str: - """ - Install a Splunkbase app. - - install_splunkbase_app(app, app_id, version, target_url, token, licence) -> status : str - """ - # Authenticate to Splunkbase - splunkbase_token = authenticate_splunkbase() - # Install the app - url = f"{target_url}?splunkbase=true" - - headers = { - 'X-Splunkbase-Authorization': splunkbase_token, - 'ACS-Licensing-Ack': licence, - 'Authorization': f'Bearer {token}', - 'Content-Type': 'application/x-www-form-urlencoded', - } - - data = { - 'splunkbaseID': app_id, - 'version': version - } - - response = requests.post(url, headers=headers, data=data) - # Handle the case where the app is already installed - if response.status_code == 409: - print(f"App {app} is already installed.") - print(f"Updating app {app} to version {version}...") - # Get app name - url = f"https://splunkbase.splunk.com/api/v1/app/{app_id}" - response = requests.get(url) - app_name = response.json().get('appid') - print(f"App name: {app_name}") - # Update the app - url = f"{target_url}/{app_name}" - data = { - 'version': version - } - response = requests.patch(url, headers=headers, data=data) - return "success - existing app updated" - elif response.ok: - request_status = response.json()['status'] - print(f"Request status: {request_status}") - if request_status in ("installed", "processing"): - print(f"App {app} version {version} installation successful.") - return "success" - else: - print(f"App {app} version {version} installation failed.") - return f"failed with status: {request_status} - {response.text}" - else: - print("Request failed!") - print(f"Status code: {response.status_code}") - print(response.text) - return f"failed with status code: {response.status_code} - {response.text}" - -def get_app_id(app_name: str) -> str: - """ - Get the Splunkbase app ID. - - get_app_id(app_name) -> app_id : str - """ - url = f"https://splunkbase.splunk.com/api/v1/app" - params = { - "query": app_name, - "limit": 1 - } - response = requests.get(url, params=params) - if len(response.json().get('results')) > 0: - app_id = response.json().get('results')[0].get('uid') - return app_id - else: - print(f"App {app_name} not found on Splunkbase.") - return None - -def get_license_url(app_name: str) -> str: - """ - Get the licence URL for a Splunkbase app. - - get_licence_url(app_name) -> licence_url : str - """ - url = f"https://splunkbase.splunk.com/api/v1/app" - params = { - "query": app_name, - "limit": 1 - } - response = requests.get(url, params=params) - if len(response.json().get('results')) > 0: - license_url = response.json().get('results')[0].get('license_url') - return license_url - else: - print(f"App {app_name} not found on Splunkbase.") - return None From caff4c7723a66072fdb13d95c898e1b9a0958f36 Mon Sep 17 00:00:00 2001 From: bdebek-splunk Date: Thu, 16 Jan 2025 11:54:04 +0100 Subject: [PATCH 02/20] docs: update README to include new modules directory and its purpose --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 3110457..aee2260 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,10 @@ Assumptions: │ ├── collections.conf │ └── logging.conf ├── deploy.py +├── modules +│ ├── apps_processing.py +│ ├── aws_s3.py +│ └── splunkcloud.py └── environments ├── prod │ ├── es @@ -38,6 +42,7 @@ Assumptions: * deployment instructions per each environment (`deployment.yml`) * specific apps configurations (e.g. `uat/es/app1`) * `deploy.py` Used by the automation to perform the deployment +* `modules/` Contains methods used in deployment automation This repository follows the same structure. Please navigate it to verify its content. From 51fe80b5e3f87dfaf7b230a27574f1ea27bc3c18 Mon Sep 17 00:00:00 2001 From: bdebek-splunk Date: Thu, 16 Jan 2025 11:59:53 +0100 Subject: [PATCH 03/20] fix: update validate_data method signature to return a dictionary and boolean values --- modules/apps_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/apps_processing.py b/modules/apps_processing.py index 2dde9bb..56393c5 100644 --- a/modules/apps_processing.py +++ b/modules/apps_processing.py @@ -25,7 +25,7 @@ def validate_data(self) -> tuple: Return boolean values for private_apps and splunkbase_apps presence in the environment configuration - validate_data(data) -> (bool, bool) + validate_data(data) -> (dict, bool, bool) """ try: data = self._read_yaml() From 93b87d25f268416c1025250d269a125e29995875 Mon Sep 17 00:00:00 2001 From: bdebek-splunk Date: Mon, 20 Jan 2025 12:46:45 +0100 Subject: [PATCH 04/20] refactor: update deployment workflow, deleted AwsS3Connector, added report_generator.py with DeploymentReportGenerator class and added DeploymentParser class in apps_processing.py --- .github/workflows/deploy.yml | 3 +- deploy.py | 77 ++++++++++------------ modules/apps_processing.py | 120 +++++++++++++++++++++++++++++------ modules/aws_s3.py | 24 ------- modules/report_generator.py | 35 ++++++++++ 5 files changed, 171 insertions(+), 88 deletions(-) delete mode 100644 modules/aws_s3.py create mode 100644 modules/report_generator.py diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 4997929..a3210a9 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -42,9 +42,10 @@ jobs: SPLUNK_USERNAME: ${{ secrets.SPLUNK_USERNAME }} SPLUNK_PASSWORD: ${{ secrets.SPLUNK_PASSWORD }} SPLUNK_TOKEN: ${{ secrets[matrix.environment.token_key] }} + DEPLOYMENT_CONFIG_PATH: "environments/${{ matrix.environment.name }}" run: | echo "Deploying to ${{ matrix.environment.name }} environment" - python -u deploy.py ${{ matrix.environment.name }} + python -u deploy.py - name: Upload deployment report as artifact uses: actions/upload-artifact@v3 with: diff --git a/deploy.py b/deploy.py index 1abcb78..8f35ac6 100644 --- a/deploy.py +++ b/deploy.py @@ -1,14 +1,15 @@ import sys import json import os +import boto3 from modules.splunkcloud import SplunkCloudConnector -from modules.aws_s3 import AwsS3Connector -from modules.apps_processing import AppFilesProcessor +from modules.apps_processing import AppFilesProcessor, DeploymentParser +from modules.report_generator import DeploymentReportGenerator # FOR LOCAL TESTING -# from dotenv import load_dotenv -# load_dotenv(dotenv_path="local.env") +from dotenv import load_dotenv +load_dotenv(dotenv_path="local.env") SPLUNK_USERNAME = os.getenv("SPLUNK_USERNAME") SPLUNK_PASSWORD = os.getenv("SPLUNK_PASSWORD") @@ -17,45 +18,43 @@ AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") +DEPLOYMENT_CONFIG_PATH = os.getenv("DEPLOYMENT_CONFIG_PATH") -def main(): - if len(sys.argv) != 2: - print("Usage: python script.py ") - sys.exit(1) - - yaml_file_path = "environments/" + sys.argv[1] + "/deployment.yml" +def main(): # Initiate deployment report - deployment_report = {} - + deployment_report = DeploymentReportGenerator() # Initiate AppFilesProcessor object - app_processor = AppFilesProcessor(yaml_file_path) + app_processor = AppFilesProcessor() + # Initiate DeploymentParser object + deployment_parser = DeploymentParser() ### 1. Validate data and retrieve all apps listed in deployment.yml from S3 ### - data, private_apps, splunkbase_apps = app_processor.validate_data() - # List all apps in yaml file and then their S3 bucket - if private_apps: - apps = data.get("apps", {}).keys() - s3_buckets = [data["apps"][app]["s3-bucket"] for app in apps] - app_directories = [data["apps"][app]["source"] for app in apps] + data, _, _ = deployment_parser.parse() target_url = data["target"]["url"] # Initiate AwsS3Connector object - s3_connector = AwsS3Connector(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) + s3_connector = boto3.client("s3") # Check for private apps - if private_apps: + if deployment_parser.has_private_apps(): print("Found private apps in deployment.yml, starting deployment...") + # List all apps in yaml file and then their S3 buckets + apps = deployment_parser.get_private_apps() + s3_buckets = deployment_parser.get_s3_buckets() + app_directories = deployment_parser.get_app_directories() # Loop through all apps for app, bucket, directory in zip(apps, s3_buckets, app_directories): object_name = directory file_name = f"{app}.tgz" # Donwload app from S3 - s3_connector.download_file_from_s3(bucket, object_name, file_name) + try: + s3_connector.download_file(bucket, object_name, file_name) + except Exception as e: + raise Exception(f"Error downloading {object_name} from {bucket}: {e}") ### 2. Upload_local_configuration ### # Check if the configuration exists for the app - path = os.path.join("environments", sys.argv[1], app) - print(path) + path = os.path.join(DEPLOYMENT_CONFIG_PATH, app) if path: app_processor.unpack_merge_conf_and_meta_repack(app, path) else: @@ -69,10 +68,10 @@ def main(): report, token = cloud_connector.cloud_validate_app(app) if report is None: print(f"App {app} failed validation.") - deployment_report[app] = {"validation": "failed"} + deployment_report.add_data(app, ("validation", "failed")) continue result = report["summary"] - deployment_report[app] = report + deployment_report.add_data(app, ("report", report)) ### 4. If app is valid, distribute it ### if ( result["error"] == 0 @@ -82,24 +81,21 @@ def main(): distribution_status = cloud_connector.distribute_app(app, token) if distribution_status == 200: print(f"App {app} successfully distributed.\n") - deployment_report[app]["distribution"] = "success" + # deployment_report[app]["distribution"] = "success" + deployment_report.add_data(app, ("distribution", "success")) else: print(f"App {app} failed distribution.") - deployment_report[app][ - "distribution" - ] = f"failed with status code: {distribution_status}" + deployment_report.add_data(app, ("distribution", f"failed with status code: {distribution_status}")) else: print(f"App {app} failed validation. Skipping distribution.\n") - deployment_report[app][ - "distribution" - ] = "failed due to app validation error" + deployment_report.add_data(app, ("distribution", "failed due to app validation error")) else: print("No private apps found in deployment.yml, skipping...") ### 5. Handle Splunkbase apps ### - if splunkbase_apps: + if deployment_parser.has_splunkbase_apps(): print("Found Splunkbase apps in deployment.yml, starting deployment...") - splunkbase_apps_dict = data.get("splunkbase-apps", {}) + splunkbase_apps_dict = deployment_parser.get_splunkbase_apps() for splunkbase_app in splunkbase_apps_dict: app = splunkbase_apps_dict[splunkbase_app] app_name = splunkbase_app @@ -110,21 +106,16 @@ def main(): app_name, app_id, version, license ) print(f"App {app_name} installation status: {install_status}") - deployment_report[app_name] = { + deployment_report.add_data(app_name, { "splunkbase_installation": install_status, "version": version, "app_id": app_id, - } + }) else: print("No Splunkbase apps found in deployment.yml, skipping...") ### 6. Save deployment report to json file ### - report_prefix = f"{sys.argv[1].split('/')[-2]}_{sys.argv[1].split('/')[-1]}" - output_dir = "artifacts" - os.makedirs(output_dir, exist_ok=True) - with open(f"{output_dir}/{report_prefix}_deployment_report.json", "w") as file: - json.dump(deployment_report, file) - + deployment_report.generate_report() if __name__ == "__main__": main() diff --git a/modules/apps_processing.py b/modules/apps_processing.py index 56393c5..ab27a59 100644 --- a/modules/apps_processing.py +++ b/modules/apps_processing.py @@ -4,31 +4,35 @@ import shutil import configparser import tarfile +import json from io import StringIO -class AppFilesProcessor: - """Class for handling local app files and configurations.""" - def __init__(self, yml_path): - self.yml_path = yml_path +class DeploymentParser: + """Class for parsing the deployment configuration file.""" - def _read_yaml(self) -> dict: - """Read and return the contents of a YAML file.""" - file_path = self.yml_path - with open(file_path, "r") as file: - return yaml.safe_load(file) - - def validate_data(self) -> tuple: + def __init__(self): + if not "DEPLOYMENT_CONFIG_PATH" in os.environ: + raise Exception(f"Error - Environment variable DEPLOYMENT_CONFIG_PATH does not exist.") + yml_path = os.path.join(os.getenv("DEPLOYMENT_CONFIG_PATH"), "deployment.yml") + # Read the file + try: + with open(yml_path, "r") as file: + self.data = yaml.safe_load(file) + except FileNotFoundError: + raise Exception(f"File not found: {yml_path}") + except yaml.YAMLError as e: + raise Exception(f"Error parsing YAML file: {e}") + + def _validate_data(self) -> None: """ Validate the data in the YAML file. - Return boolean values for private_apps and splunkbase_apps presence in the environment configuration - - validate_data(data) -> (dict, bool, bool) + validate_data(data) -> None """ try: - data = self._read_yaml() + data = self.data except FileNotFoundError: print(f"Error: The file '{self.yml_path}' was not found.") except yaml.YAMLError as e: @@ -48,13 +52,89 @@ def validate_data(self) -> tuple: print("Error: The 'splunkbase-apps' key is missing.") sys.exit(1) - app_dict = data.get("apps", {}) - splunkbase_dict = data.get("splunkbase-apps", {}) + def parse(self) -> tuple: + """ + Return the parsed data from the deployment configuration file. + + parse() -> (dict, dict, dict) + """ + self._validate_data() + private_apps_dict = self.data.get("apps", {}) + splunkbase_dict = self.data.get("splunkbase-apps", {}) + + return self.data, private_apps_dict, splunkbase_dict + + def has_private_apps(self) -> bool: + """ + Check if private apps are present in the deployment configuration. + + has_private_apps() -> bool + """ + private_apps = self.parse()[1] + return True if private_apps else False + + def has_splunkbase_apps(self) -> bool: + """ + Check if Splunkbase apps are present in the deployment configuration. + + has_splunkbase_apps() -> bool + """ + splunkbase_apps = self.parse()[2] + return True if splunkbase_apps else False + + def get_private_apps(self) -> list: + """ + Return a dictionary of private apps from the deployment configuration. - private_apps = True if app_dict else False - splunkbase_apps = True if splunkbase_dict else False + get_apps() -> list + """ + private_apps = self.parse()[1] + return private_apps.keys() + + def get_s3_buckets(self) -> list: + """ + Return a list of S3 buckets from the deployment configuration. - return data, private_apps, splunkbase_apps + s3_buckets() -> list + """ + data = self.parse()[0] + apps = self.get_private_apps() + return [data["apps"][app]["s3-bucket"] for app in apps] + + def get_app_directories(self) -> list: + """ + Return a list of app directories from the deployment configuration. + + get_app_directories() -> list + """ + data = self.parse()[0] + apps = self.get_private_apps() + return [data["apps"][app]["source"] for app in apps] + + def get_splunkbase_apps(self) -> dict: + """ + Return a dictionary of Splunkbase apps from the deployment configuration. + + get_splunkbase_apps() -> dict + """ + splunkbase_apps = self.parse()[2] + return splunkbase_apps + +class AppFilesProcessor: + """Class for handling local app files and configurations.""" + + def __init__(self): + if not "DEPLOYMENT_CONFIG_PATH" in os.environ: + raise Exception(f"Error - Environment variable DEPLOYMENT_CONFIG_PATH does not exist.") + yml_path = os.path.join(os.getenv("DEPLOYMENT_CONFIG_PATH"), "deployment.yml") + # Read the file + try: + with open(yml_path, "r") as file: + self.data = yaml.safe_load(file) + except FileNotFoundError: + raise Exception(f"File not found: {yml_path}") + except yaml.YAMLError as e: + raise Exception(f"Error parsing YAML file: {e}") def _preprocess_empty_headers(self, file_path: str) -> list: """ diff --git a/modules/aws_s3.py b/modules/aws_s3.py deleted file mode 100644 index 78b4faa..0000000 --- a/modules/aws_s3.py +++ /dev/null @@ -1,24 +0,0 @@ -import boto3 - - -class AwsS3Connector: - """Class to connect to AWS S3 and download files.""" - - def __init__(self, aws_access_key_id, aws_secret_access_key): - self.aws_access_key_id = aws_access_key_id - self.aws_secret_access_key = aws_secret_access_key - - def download_file_from_s3( - self, bucket_name: str, object_name: str, file_name: str - ) -> None: - """Download a file from an S3 bucket.""" - s3 = boto3.client( - "s3", - aws_access_key_id=self.aws_access_key_id, - aws_secret_access_key=self.aws_secret_access_key, - ) - try: - s3.download_file(bucket_name, object_name, file_name) - print(f"Downloaded {object_name} from {bucket_name} to {file_name}") - except Exception as e: - print(f"Error downloading {object_name} from {bucket_name}: {e}") diff --git a/modules/report_generator.py b/modules/report_generator.py new file mode 100644 index 0000000..bc373ce --- /dev/null +++ b/modules/report_generator.py @@ -0,0 +1,35 @@ +import os +import json + +class DeploymentReportGenerator: + """Class for generating deployment report.""" + + def __init__(self): + self.deployment_report = {} + + def __str__(self) -> str: + return str(self.deployment_report) + + def add_data(self, key: str, value: tuple|dict) -> None: + """Add data to deployment report.""" + deployment_report = self.deployment_report + if key not in deployment_report: + deployment_report[key] = {} + # Handle situation if passed value is a dictionary + if isinstance(value, dict): + deployment_report[key].update(value) + # Handle situation if passed value is a tuple + elif isinstance(value, tuple): + deployment_report[key][value[0]] = value[1] + else: + raise ValueError("Value must be a tuple or a dictionary.") + + def generate_report(self) -> None: + """Generate deployment report.""" + DEPLOYMENT_CONFIG_PATH = os.getenv("DEPLOYMENT_CONFIG_PATH") + report_prefix = f"{DEPLOYMENT_CONFIG_PATH.split('/')[-2]}_{DEPLOYMENT_CONFIG_PATH.split('/')[-1]}" + output_dir = "artifacts" + + os.makedirs(output_dir, exist_ok=True) + with open(f"{output_dir}/{report_prefix}_deployment_report.json", "w") as file: + json.dump(self.deployment_report, file) \ No newline at end of file From bdd94cc36871deef7f188d0b1d178dc905d16fcb Mon Sep 17 00:00:00 2001 From: bdebek-splunk Date: Mon, 20 Jan 2025 12:47:06 +0100 Subject: [PATCH 05/20] fix: update deployment report to use add_data method for successful app distribution --- deploy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy.py b/deploy.py index 8f35ac6..807bd95 100644 --- a/deploy.py +++ b/deploy.py @@ -81,7 +81,6 @@ def main(): distribution_status = cloud_connector.distribute_app(app, token) if distribution_status == 200: print(f"App {app} successfully distributed.\n") - # deployment_report[app]["distribution"] = "success" deployment_report.add_data(app, ("distribution", "success")) else: print(f"App {app} failed distribution.") From 05dbeb94558d04a488d30229a945c4cd1d33d308 Mon Sep 17 00:00:00 2001 From: bdebek-splunk Date: Mon, 20 Jan 2025 12:50:16 +0100 Subject: [PATCH 06/20] fix: added report_generator.py to the architectire tree --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aee2260..89df31c 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Assumptions: ├── deploy.py ├── modules │ ├── apps_processing.py -│ ├── aws_s3.py +│ ├── report_generatot.py │ └── splunkcloud.py └── environments ├── prod From e69d682efdd560fce7462ceee0209d1d090d5d19 Mon Sep 17 00:00:00 2001 From: bdebek-splunk Date: Mon, 20 Jan 2025 13:41:28 +0100 Subject: [PATCH 07/20] style: clean up code formatting and improve type hinting in deployment modules --- deploy.py | 38 ++++++++++++++++++++++++------------- modules/apps_processing.py | 38 ++++++++++++++++++++----------------- modules/report_generator.py | 6 ++++-- 3 files changed, 50 insertions(+), 32 deletions(-) diff --git a/deploy.py b/deploy.py index 807bd95..f1155f7 100644 --- a/deploy.py +++ b/deploy.py @@ -8,8 +8,8 @@ from modules.report_generator import DeploymentReportGenerator # FOR LOCAL TESTING -from dotenv import load_dotenv -load_dotenv(dotenv_path="local.env") +# from dotenv import load_dotenv +# load_dotenv(dotenv_path="local.env") SPLUNK_USERNAME = os.getenv("SPLUNK_USERNAME") SPLUNK_PASSWORD = os.getenv("SPLUNK_PASSWORD") @@ -34,7 +34,7 @@ def main(): target_url = data["target"]["url"] # Initiate AwsS3Connector object - s3_connector = boto3.client("s3") + s3_connector = boto3.client("s3") # Check for private apps if deployment_parser.has_private_apps(): print("Found private apps in deployment.yml, starting deployment...") @@ -47,10 +47,10 @@ def main(): object_name = directory file_name = f"{app}.tgz" # Donwload app from S3 - try: + try: s3_connector.download_file(bucket, object_name, file_name) - except Exception as e: - raise Exception(f"Error downloading {object_name} from {bucket}: {e}") + except Exception as e: + raise Exception(f"Error downloading {object_name} from {bucket}: {e}") ### 2. Upload_local_configuration ### # Check if the configuration exists for the app @@ -84,10 +84,18 @@ def main(): deployment_report.add_data(app, ("distribution", "success")) else: print(f"App {app} failed distribution.") - deployment_report.add_data(app, ("distribution", f"failed with status code: {distribution_status}")) + deployment_report.add_data( + app, + ( + "distribution", + f"failed with status code: {distribution_status}", + ), + ) else: print(f"App {app} failed validation. Skipping distribution.\n") - deployment_report.add_data(app, ("distribution", "failed due to app validation error")) + deployment_report.add_data( + app, ("distribution", "failed due to app validation error") + ) else: print("No private apps found in deployment.yml, skipping...") @@ -105,16 +113,20 @@ def main(): app_name, app_id, version, license ) print(f"App {app_name} installation status: {install_status}") - deployment_report.add_data(app_name, { - "splunkbase_installation": install_status, - "version": version, - "app_id": app_id, - }) + deployment_report.add_data( + app_name, + { + "splunkbase_installation": install_status, + "version": version, + "app_id": app_id, + }, + ) else: print("No Splunkbase apps found in deployment.yml, skipping...") ### 6. Save deployment report to json file ### deployment_report.generate_report() + if __name__ == "__main__": main() diff --git a/modules/apps_processing.py b/modules/apps_processing.py index ab27a59..e6653c4 100644 --- a/modules/apps_processing.py +++ b/modules/apps_processing.py @@ -8,13 +8,14 @@ from io import StringIO - class DeploymentParser: """Class for parsing the deployment configuration file.""" def __init__(self): if not "DEPLOYMENT_CONFIG_PATH" in os.environ: - raise Exception(f"Error - Environment variable DEPLOYMENT_CONFIG_PATH does not exist.") + raise Exception( + f"Error - Environment variable DEPLOYMENT_CONFIG_PATH does not exist." + ) yml_path = os.path.join(os.getenv("DEPLOYMENT_CONFIG_PATH"), "deployment.yml") # Read the file try: @@ -24,7 +25,7 @@ def __init__(self): raise Exception(f"File not found: {yml_path}") except yaml.YAMLError as e: raise Exception(f"Error parsing YAML file: {e}") - + def _validate_data(self) -> None: """ Validate the data in the YAML file. @@ -52,15 +53,15 @@ def _validate_data(self) -> None: print("Error: The 'splunkbase-apps' key is missing.") sys.exit(1) - def parse(self) -> tuple: - """ - Return the parsed data from the deployment configuration file. + def parse(self) -> tuple: + """ + Return the parsed data from the deployment configuration file. - parse() -> (dict, dict, dict) - """ - self._validate_data() - private_apps_dict = self.data.get("apps", {}) - splunkbase_dict = self.data.get("splunkbase-apps", {}) + parse() -> (dict, dict, dict) + """ + self._validate_data() + private_apps_dict = self.data.get("apps", {}) + splunkbase_dict = self.data.get("splunkbase-apps", {}) return self.data, private_apps_dict, splunkbase_dict @@ -72,7 +73,7 @@ def has_private_apps(self) -> bool: """ private_apps = self.parse()[1] return True if private_apps else False - + def has_splunkbase_apps(self) -> bool: """ Check if Splunkbase apps are present in the deployment configuration. @@ -90,7 +91,7 @@ def get_private_apps(self) -> list: """ private_apps = self.parse()[1] return private_apps.keys() - + def get_s3_buckets(self) -> list: """ Return a list of S3 buckets from the deployment configuration. @@ -100,7 +101,7 @@ def get_s3_buckets(self) -> list: data = self.parse()[0] apps = self.get_private_apps() return [data["apps"][app]["s3-bucket"] for app in apps] - + def get_app_directories(self) -> list: """ Return a list of app directories from the deployment configuration. @@ -110,7 +111,7 @@ def get_app_directories(self) -> list: data = self.parse()[0] apps = self.get_private_apps() return [data["apps"][app]["source"] for app in apps] - + def get_splunkbase_apps(self) -> dict: """ Return a dictionary of Splunkbase apps from the deployment configuration. @@ -119,13 +120,16 @@ def get_splunkbase_apps(self) -> dict: """ splunkbase_apps = self.parse()[2] return splunkbase_apps - + + class AppFilesProcessor: """Class for handling local app files and configurations.""" def __init__(self): if not "DEPLOYMENT_CONFIG_PATH" in os.environ: - raise Exception(f"Error - Environment variable DEPLOYMENT_CONFIG_PATH does not exist.") + raise Exception( + f"Error - Environment variable DEPLOYMENT_CONFIG_PATH does not exist." + ) yml_path = os.path.join(os.getenv("DEPLOYMENT_CONFIG_PATH"), "deployment.yml") # Read the file try: diff --git a/modules/report_generator.py b/modules/report_generator.py index bc373ce..e7d86ed 100644 --- a/modules/report_generator.py +++ b/modules/report_generator.py @@ -1,5 +1,7 @@ import os import json +from typing import Union + class DeploymentReportGenerator: """Class for generating deployment report.""" @@ -10,7 +12,7 @@ def __init__(self): def __str__(self) -> str: return str(self.deployment_report) - def add_data(self, key: str, value: tuple|dict) -> None: + def add_data(self, key: str, value: Union[tuple, dict]) -> None: """Add data to deployment report.""" deployment_report = self.deployment_report if key not in deployment_report: @@ -32,4 +34,4 @@ def generate_report(self) -> None: os.makedirs(output_dir, exist_ok=True) with open(f"{output_dir}/{report_prefix}_deployment_report.json", "w") as file: - json.dump(self.deployment_report, file) \ No newline at end of file + json.dump(self.deployment_report, file) From e88b721340fd772f9e3964b6bf1cbbce6b8bd7a2 Mon Sep 17 00:00:00 2001 From: bdebek-splunk Date: Mon, 20 Jan 2025 15:22:29 +0100 Subject: [PATCH 08/20] refactor: removed hardcoded endpoint from instance url defined in deployment.yml --- deploy.py | 4 ++-- environments/uat/es/deployment.yml | 2 +- environments/uat/ses/deployment.yml | 2 +- modules/splunkcloud.py | 11 +++++++---- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/deploy.py b/deploy.py index f1155f7..3c76523 100644 --- a/deploy.py +++ b/deploy.py @@ -8,8 +8,8 @@ from modules.report_generator import DeploymentReportGenerator # FOR LOCAL TESTING -# from dotenv import load_dotenv -# load_dotenv(dotenv_path="local.env") +from dotenv import load_dotenv +load_dotenv(dotenv_path="local.env") SPLUNK_USERNAME = os.getenv("SPLUNK_USERNAME") SPLUNK_PASSWORD = os.getenv("SPLUNK_PASSWORD") diff --git a/environments/uat/es/deployment.yml b/environments/uat/es/deployment.yml index 522eaf8..4a14cae 100644 --- a/environments/uat/es/deployment.yml +++ b/environments/uat/es/deployment.yml @@ -1,5 +1,5 @@ target: - url: https://staging.admin.splunk.com/scv-shw-217bd09bcbf264/adminconfig/v2/apps/victoria + url: https://staging.admin.splunk.com/scv-shw-a7f6020a334e01 apps: Splunk_TA_app1: s3-bucket: splunk-apps-deployment diff --git a/environments/uat/ses/deployment.yml b/environments/uat/ses/deployment.yml index 75ba15b..1d8d38f 100644 --- a/environments/uat/ses/deployment.yml +++ b/environments/uat/ses/deployment.yml @@ -1,5 +1,5 @@ target: - url: https://staging.admin.splunk.com/scv-shw-d037e758abafa2/adminconfig/v2/apps/victoria + url: https://staging.admin.splunk.com/scv-shw-d037e758abafa2 apps: Splunk_TA_app1: s3-bucket: splunk-apps-deployment diff --git a/modules/splunkcloud.py b/modules/splunkcloud.py index 0153ced..65b7885 100644 --- a/modules/splunkcloud.py +++ b/modules/splunkcloud.py @@ -13,6 +13,8 @@ class SplunkCloudConnector: SPLUNKBASE_BASE_URL = "https://splunkbase.splunk.com/api/account:login" SPLUNKBASE_APP_URL = "https://splunkbase.splunk.com/api/v1/app" + SPLUNK_CLOUD_APP_INSTALL_ENDPOINT = "/adminconfig/v2/apps/victoria" + def __init__( self, splunk_username: str = None, @@ -125,8 +127,8 @@ def distribute_app(self, app: str, token: str) -> int: distribute_app(app, target_url, token) -> status_code : int """ print(f"Distributing {app} to {self.splunk_host}") - target_url = self.splunk_host - url = target_url + base_url = self.splunk_host + url = base_url + self.SPLUNK_CLOUD_APP_INSTALL_ENDPOINT admin_token = self.splunk_token headers = { "X-Splunk-Authorization": token, @@ -138,7 +140,7 @@ def distribute_app(self, app: str, token: str) -> int: with open(file_path, "rb") as file: response = requests.post(url, headers=headers, data=file) print( - f"Distributed {app} to {target_url} with response: {response.status_code} {response.text}" + f"Distributed {app} to {base_url} with response: {response.status_code} {response.text}" ) except Exception as e: print(f"Error distributing {app} to {self.splunk_host}: {e}") @@ -182,7 +184,8 @@ def install_splunkbase_app( # Authenticate to Splunkbase splunkbase_token = self.authenticate_splunkbase() # Install the app - target_url = self.splunk_host + base_url = self.splunk_host + target_url = base_url + self.SPLUNK_CLOUD_APP_INSTALL_ENDPOINT token = self.splunk_token url = f"{target_url}?splunkbase=true" From 08c3f0e14039b78379515824b69268e315f054bc Mon Sep 17 00:00:00 2001 From: bdebek-splunk Date: Mon, 20 Jan 2025 16:52:32 +0100 Subject: [PATCH 09/20] refactor: reworked splunkcloud.py and deploy.py modules --- deploy.py | 74 ++++------ modules/splunkcloud.py | 307 ++++++++++++++++++++--------------------- 2 files changed, 173 insertions(+), 208 deletions(-) diff --git a/deploy.py b/deploy.py index 3c76523..24bfd2a 100644 --- a/deploy.py +++ b/deploy.py @@ -1,5 +1,3 @@ -import sys -import json import os import boto3 @@ -7,14 +5,6 @@ from modules.apps_processing import AppFilesProcessor, DeploymentParser from modules.report_generator import DeploymentReportGenerator -# FOR LOCAL TESTING -from dotenv import load_dotenv -load_dotenv(dotenv_path="local.env") - -SPLUNK_USERNAME = os.getenv("SPLUNK_USERNAME") -SPLUNK_PASSWORD = os.getenv("SPLUNK_PASSWORD") -SPLUNK_TOKEN = os.getenv("SPLUNK_TOKEN") - AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") @@ -28,7 +18,6 @@ def main(): app_processor = AppFilesProcessor() # Initiate DeploymentParser object deployment_parser = DeploymentParser() - ### 1. Validate data and retrieve all apps listed in deployment.yml from S3 ### data, _, _ = deployment_parser.parse() target_url = data["target"]["url"] @@ -62,40 +51,33 @@ def main(): ### 3. Validate app for Splunk Cloud ### # Initiate SplunkCloudConnector object - cloud_connector = SplunkCloudConnector( - SPLUNK_USERNAME, SPLUNK_PASSWORD, SPLUNK_TOKEN, target_url - ) - report, token = cloud_connector.cloud_validate_app(app) - if report is None: - print(f"App {app} failed validation.") + cloud_connector = SplunkCloudConnector(target_url) + appinspect_handler = cloud_connector.get_appinspect_handler() + is_valid = appinspect_handler.validate(app) + if not is_valid: + print(f"App {app} failed validation. Skipping distribution.\n") + deployment_report.add_data(app, ("report", appinspect_handler.report)) deployment_report.add_data(app, ("validation", "failed")) - continue - result = report["summary"] - deployment_report.add_data(app, ("report", report)) + deployment_report.add_data( + app, ("distribution", "failed due to app validation error") + ) + # App is valid + deployment_report.add_data(app, ("report", appinspect_handler.report)) ### 4. If app is valid, distribute it ### - if ( - result["error"] == 0 - and result["failure"] == 0 - and result["manual_check"] == 0 - ): - distribution_status = cloud_connector.distribute_app(app, token) - if distribution_status == 200: - print(f"App {app} successfully distributed.\n") - deployment_report.add_data(app, ("distribution", "success")) - else: - print(f"App {app} failed distribution.") - deployment_report.add_data( - app, - ( - "distribution", - f"failed with status code: {distribution_status}", - ), - ) + dist_succeeded, dist_status = cloud_connector.distribute(app) + if dist_succeeded: + print(f"App {app} successfully distributed.\n") + deployment_report.add_data(app, ("distribution", "success")) else: - print(f"App {app} failed validation. Skipping distribution.\n") + print(f"App {app} failed distribution.") deployment_report.add_data( - app, ("distribution", "failed due to app validation error") + app, + ( + "distribution", + f"failed with status code: {dist_status}", + ), ) + else: print("No private apps found in deployment.yml, skipping...") @@ -107,19 +89,11 @@ def main(): app = splunkbase_apps_dict[splunkbase_app] app_name = splunkbase_app version = app["version"] - app_id = cloud_connector.get_app_id(app_name) - license = cloud_connector.get_license_url(app_name) - install_status = cloud_connector.install_splunkbase_app( - app_name, app_id, version, license - ) + install_status = cloud_connector.install(app_name, version) print(f"App {app_name} installation status: {install_status}") deployment_report.add_data( app_name, - { - "splunkbase_installation": install_status, - "version": version, - "app_id": app_id, - }, + {"splunkbase_installation": install_status, "version": version}, ) else: print("No Splunkbase apps found in deployment.yml, skipping...") diff --git a/modules/splunkcloud.py b/modules/splunkcloud.py index 65b7885..95b8ea6 100644 --- a/modules/splunkcloud.py +++ b/modules/splunkcloud.py @@ -4,49 +4,56 @@ import xml.etree.ElementTree as ET +# TODO remove following. Used for local testing only. +# from dotenv import load_dotenv +# load_dotenv(dotenv_path="local.env") + + +class SplunkCloudAccountConfig: + username: str = os.getenv("SPLUNK_USERNAME") + password: str = os.getenv("SPLUNK_PASSWORD") + token: str = os.getenv("SPLUNK_TOKEN") + + @classmethod + def to_dict(cls) -> dict: + return cls.__dict__ -class SplunkCloudConnector: - """Class for connecting to Splunk Cloud and Splunkbase.""" - SPLUNK_AUTH_BASE_URL = "https://api.splunk.com/2.0/rest/login/splunk" - SPLUNK_APPINSPECT_BASE_URL = "https://appinspect.splunk.com/v1" - SPLUNKBASE_BASE_URL = "https://splunkbase.splunk.com/api/account:login" - SPLUNKBASE_APP_URL = "https://splunkbase.splunk.com/api/v1/app" - - SPLUNK_CLOUD_APP_INSTALL_ENDPOINT = "/adminconfig/v2/apps/victoria" - - def __init__( - self, - splunk_username: str = None, - splunk_password: str = None, - splunk_token: str = None, - splunk_host: str = None, - ): - self.splunk_username = splunk_username - self.splunk_password = splunk_password - self.splunk_token = splunk_token - self.splunk_host = splunk_host - - def get_appinspect_token(self) -> str: +class AppInspectService: + base_url: str = "https://appinspect.splunk.com/v1" + auth_url: str = "https://api.splunk.com/2.0/rest/login/splunk" + report: dict = {} + tags: str = "private_victoria" + + def __init__(self, cloud_type: str = "victoria"): + self.account = SplunkCloudAccountConfig.to_dict() + self.tags = f"private_{cloud_type}" + + def get_token(self) -> str: """ Authenticate to the Splunk Cloud. - get_appinspect_token() -> token : str + get_token() -> token : str """ - url = self.SPLUNK_AUTH_BASE_URL - username = self.splunk_username - password = self.splunk_password - - response = requests.get(url, auth=(username, password)) - token = response.json()["data"]["token"] - return token + try: + response = requests.get( + self.auth_url, auth=(self.account["username"], self.account["password"]) + ) + token = response.json()["data"]["token"] + print("AppInspectService: get_token() - success") + return token + except requests.exceptions.RequestException as e: + print(f"Error getting token: {e}") + return None - def validation_request_helper(self, url: str, headers: dict, files: dict) -> str: + def _get_request_id(self, headers: dict, files: dict) -> str: """ Helper function to make a validation request and return the request ID. - validation_request_helper(url, headers, files) -> request_id : str + _get_request_id(headers, files) -> request_id : str """ + url = f"{self.base_url}/app/validate" + try: response = requests.post(url, headers=headers, files=files, timeout=120) response_json = response.json() @@ -56,108 +63,92 @@ def validation_request_helper(self, url: str, headers: dict, files: dict) -> str return None return request_id - def cloud_validate_app(self, app: str) -> tuple: + def validate(self, app: str) -> bool: """ Validate the app for the Splunk Cloud. - cloud_validate_app(app) -> report : dict, token : str + validate(app) -> is_valid: bool """ - token = self.get_appinspect_token() - base_url = self.SPLUNK_APPINSPECT_BASE_URL - url = f"{base_url}/app/validate" - + token = self.get_token() headers = {"Authorization": f"Bearer {token}"} - app_file_path = f"{app}.tgz" print(f"Validating app {app}...") - with open(app_file_path, "rb") as file: - files = {"app_package": file} - request_id = self.validation_request_helper(url, headers, files) - headers = {"Authorization": f"Bearer {token}"} - status_url = f"{base_url}/app/validate/status/{request_id}?included_tags=private_victoria" + with open(f"{app}.tgz", "rb") as file: + request_id = self._get_request_id(headers, {"app_package": file}) + status_url = f"{self.base_url}/app/validate/status/{request_id}?included_tags={self.tags}" + try: - response_status = requests.get(status_url, headers=headers) + response = requests.get(status_url, headers=headers) except requests.exceptions.RequestException as e: print(f"Error: {e}") return None, None max_retries = 60 # Maximum number of retries retries = 0 - response_status_json = response_status.json() + response_json = response.json() - while response_status_json["status"] != "SUCCESS" and retries < max_retries: - response_status = requests.get(status_url, headers=headers) - response_status_json = response_status.json() + while response_json["status"] != "SUCCESS" and retries < max_retries: + response = requests.get(status_url, headers=headers) + response_json = response.json() retries += 1 - if response_status_json["status"] == "FAILURE": - print( - f"App {app} failed validation: {response_status_json['errors']}" - ) + if response_json["status"] == "FAILURE": + print(f"App {app} failed validation: {response_json['errors']}") break else: print(f"App {app} awaiting validation...") - print(f"Current status: {response_status_json['status']}") + print(f"Current status: {response_json['status']}") time.sleep(10) - response_status = requests.get(status_url, headers=headers) - response_status_json = response_status.json() - continue if retries == max_retries: print(f"App {app} validation timed out.") return - print(f"Current status: {response_status_json['status']}") - if response_status_json["status"] == "SUCCESS": + print(f"Current status: {response_json['status']}. Fetching summary.") + if response_json["status"] == "SUCCESS": print("App validation successful.") - print("Installing app...") - response_report = requests.get( - f"{base_url}/app/report/{request_id}?included_tags=private_victoria", + response = requests.get( + f"{self.base_url}/app/report/{request_id}?included_tags=private_victoria", headers=headers, ) - report = response_report.json() - result = report["summary"] - print(result) + self.report = response.json() + summary = self.report["summary"] - return report, token + return ( + summary["error"] == 0 + and summary["failure"] == 0 + and summary["manual_check"] == 0 + ) - def distribute_app(self, app: str, token: str) -> int: - """ - Distribute the app to the target URL. - distribute_app(app, target_url, token) -> status_code : int - """ - print(f"Distributing {app} to {self.splunk_host}") - base_url = self.splunk_host - url = base_url + self.SPLUNK_CLOUD_APP_INSTALL_ENDPOINT - admin_token = self.splunk_token - headers = { - "X-Splunk-Authorization": token, - "Authorization": f"Bearer {admin_token}", - "ACS-Legal-Ack": "Y", - } - file_path = f"{app}.tgz" - try: - with open(file_path, "rb") as file: - response = requests.post(url, headers=headers, data=file) - print( - f"Distributed {app} to {base_url} with response: {response.status_code} {response.text}" - ) - except Exception as e: - print(f"Error distributing {app} to {self.splunk_host}: {e}") - return 500 +class SplunkbaseService: + base_url: str = "https://splunkbase.splunk.com/api/v1/app" + auth_url: str = "https://splunkbase.splunk.com/api/account:login" + token: str = None - return response.status_code + def __init__(self): + self.account = SplunkCloudAccountConfig.to_dict() + + def get_app_info(self, app_name: str) -> dict: + params = {"query": app_name, "limit": 1} + response = requests.get(self.base_url, params=params) + if len(response.json().get("results")) > 0: + return response.json().get("results")[0] + else: + print(f"App {app_name} not found on Splunkbase.") + return None - def authenticate_splunkbase(self) -> str: + def _authenticate(self) -> None: """ Authenticate to Splunkbase. - authenticate_splunkbase() -> token : str + _authenticate() -> token : str """ - url = self.SPLUNKBASE_BASE_URL - data = {"username": self.splunk_username, "password": self.splunk_password} - response = requests.post(url, data=data) + data = { + "username": self.account["username"], + "password": self.account["password"], + } + response = requests.post(self.auth_url, data=data) if response.ok: # Parse the XML response xml_root = ET.fromstring(response.text) @@ -166,54 +157,86 @@ def authenticate_splunkbase(self) -> str: splunkbase_token = xml_root.find( "atom:id", namespace ).text # Find the tag with the namespace - return splunkbase_token + self.token = splunkbase_token else: print("Splunkbase login failed!") print(f"Status code: {response.status_code}") print(response.text) - return None - def install_splunkbase_app( - self, app: str, app_id: str, version: str, licence: str - ) -> str: + def get_token(self): + if not self.token: + self._authenticate() + return self.token + + +class SplunkCloudConnector: + """Class for connecting to Splunk Cloud and Splunkbase.""" + + def __init__(self, splunk_host: str = None, cloud_type: str = "victoria"): + self.config = SplunkCloudAccountConfig.to_dict() + self.appinspect = AppInspectService() + self.splunkbase = SplunkbaseService() + self.host = splunk_host + if cloud_type == "classic": + cloud_type = "" + self.cloud_type = f"/{cloud_type}" + + def get_appinspect_handler(self): + return self.appinspect + + def distribute(self, app: str) -> tuple: """ - Install a Splunkbase app. + Distribute a private app to the target URL. - install_splunkbase_app(app, app_id, version, target_url, token, licence) -> status : str + distribute(app) -> was_successful : bool, status_code: int """ - # Authenticate to Splunkbase - splunkbase_token = self.authenticate_splunkbase() - # Install the app - base_url = self.splunk_host - target_url = base_url + self.SPLUNK_CLOUD_APP_INSTALL_ENDPOINT - token = self.splunk_token + url = f"{self.host}/adminconfig/v2/apps{self.cloud_type}" + print(f"Distributing {app} to {url}") + headers = { + "X-Splunk-Authorization": self.appinspect.get_token(), + "Authorization": f"Bearer {self.config.get('token')}", + "ACS-Legal-Ack": "Y", + } + try: + with open(f"{app}.tgz", "rb") as file: + response = requests.post(url, headers=headers, data=file) + print(f"Distributed {app} to {url} with response: {response.status_code}") + except Exception as e: + print(f"Error distributing {app} to {url}: {e}") + return False, 500 - url = f"{target_url}?splunkbase=true" + return response.status_code == 200, response.status_code + def install(self, app: str, version: str) -> str: + """ + Install a Splunkbase app. + + install(app, version) -> status : str + """ + token = self.splunkbase.get_token() + url = f"{self.host}/adminconfig/v2/apps{self.cloud_type}?splunkbase=true" + app_info = self.splunkbase.get_app_info(app) headers = { - "X-Splunkbase-Authorization": splunkbase_token, - "ACS-Licensing-Ack": licence, - "Authorization": f"Bearer {token}", + "X-Splunkbase-Authorization": token, + "ACS-Licensing-Ack": app_info.get("license_url"), + "Authorization": f"Bearer {self.config.get('token')}", "Content-Type": "application/x-www-form-urlencoded", } - data = {"splunkbaseID": app_id, "version": version} + data = {"splunkbaseID": app_info.get("uid"), "version": version} response = requests.post(url, headers=headers, data=data) # Handle the case where the app is already installed if response.status_code == 409: print(f"App {app} is already installed.") - print(f"Updating app {app} to version {version}...") - # Get app name - url = f"https://splunkbase.splunk.com/api/v1/app/{app_id}" - response = requests.get(url) - app_name = response.json().get("appid") - print(f"App name: {app_name}") + app_name = app_info.get("appid") + print(f"Updating app {app} ({app_name}) to version {version}...") # Update the app - url = f"{target_url}/{app_name}" + url = f"{self.host}/{app_name}" data = {"version": version} response = requests.patch(url, headers=headers, data=data) return "success - existing app updated" - elif response.ok: + + if response.ok: request_status = response.json()["status"] print(f"Request status: {request_status}") if request_status in ("installed", "processing"): @@ -222,40 +245,8 @@ def install_splunkbase_app( else: print(f"App {app} version {version} installation failed.") return f"failed with status: {request_status} - {response.text}" - else: - print("Request failed!") - print(f"Status code: {response.status_code}") - print(response.text) - return f"failed with status code: {response.status_code} - {response.text}" - - def get_app_id(self, app_name: str) -> str: - """ - Get the Splunkbase app ID. - - get_app_id(app_name) -> app_id : str - """ - url = self.SPLUNKBASE_APP_URL - params = {"query": app_name, "limit": 1} - response = requests.get(url, params=params) - if len(response.json().get("results")) > 0: - app_id = response.json().get("results")[0].get("uid") - return app_id - else: - print(f"App {app_name} not found on Splunkbase.") - return None - - def get_license_url(self, app_name: str) -> str: - """ - Get the licence URL for a Splunkbase app. - get_licence_url(app_name) -> licence_url : str - """ - url = self.SPLUNKBASE_APP_URL - params = {"query": app_name, "limit": 1} - response = requests.get(url, params=params) - if len(response.json().get("results")) > 0: - license_url = response.json().get("results")[0].get("license_url") - return license_url - else: - print(f"App {app_name} not found on Splunkbase.") - return None + print("Request failed!") + print(f"Status code: {response.status_code}") + print(response.text) + return f"failed with status code: {response.status_code} - {response.text}" From c8d4813ea828e280815a1265b60bceb16867f584 Mon Sep 17 00:00:00 2001 From: bdebek-splunk Date: Mon, 20 Jan 2025 16:55:30 +0100 Subject: [PATCH 10/20] fix: remove redundant status print statement in AppInspectService --- modules/splunkcloud.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/splunkcloud.py b/modules/splunkcloud.py index 95b8ea6..71f62bf 100644 --- a/modules/splunkcloud.py +++ b/modules/splunkcloud.py @@ -102,7 +102,6 @@ def validate(self, app: str) -> bool: print(f"App {app} validation timed out.") return - print(f"Current status: {response_json['status']}. Fetching summary.") if response_json["status"] == "SUCCESS": print("App validation successful.") From ec1c6217fb5dcc83e7103f9a3e6895175a7cec84 Mon Sep 17 00:00:00 2001 From: Erica Pescio Date: Tue, 21 Jan 2025 10:01:52 +0100 Subject: [PATCH 11/20] docs:updated README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 89df31c..12fe308 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Assumptions: ├── deploy.py ├── modules │ ├── apps_processing.py -│ ├── report_generatot.py +│ ├── report_generator.py │ └── splunkcloud.py └── environments ├── prod @@ -51,6 +51,7 @@ As mentioned, these deployment files specify the apps and configurations needed ```yml target: url: + experience: apps: # Private apps app1: From b2faa378e5ca4a1f1136a7426dc436c18a281343 Mon Sep 17 00:00:00 2001 From: Erica Pescio Date: Tue, 21 Jan 2025 10:03:43 +0100 Subject: [PATCH 12/20] chore:updated deployment config files to support experience key --- environments/prod/es/deployment.yml | 1 + environments/prod/ses/deployment.yml | 4 +++- environments/uat/es/deployment.yml | 2 +- environments/uat/ses/deployment.yml | 5 +++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/environments/prod/es/deployment.yml b/environments/prod/es/deployment.yml index 96aca8a..8d7522d 100644 --- a/environments/prod/es/deployment.yml +++ b/environments/prod/es/deployment.yml @@ -1,5 +1,6 @@ target: url: https://splunk-es-prod.example.com + experience: classic apps: app1: s3-bucket: splunk-apps-deployment diff --git a/environments/prod/ses/deployment.yml b/environments/prod/ses/deployment.yml index 3c57cb4..70159ea 100644 --- a/environments/prod/ses/deployment.yml +++ b/environments/prod/ses/deployment.yml @@ -1,6 +1,8 @@ target: url: + experience: apps: app1: s3-bucket: splunk-apps-deployment - source: apps/Splunk_TA_app1.tgz \ No newline at end of file + source: apps/Splunk_TA_app1.tgz +splunkbase-apps: diff --git a/environments/uat/es/deployment.yml b/environments/uat/es/deployment.yml index 4a14cae..8d5e980 100644 --- a/environments/uat/es/deployment.yml +++ b/environments/uat/es/deployment.yml @@ -1,5 +1,6 @@ target: url: https://staging.admin.splunk.com/scv-shw-a7f6020a334e01 + experience: victoria apps: Splunk_TA_app1: s3-bucket: splunk-apps-deployment @@ -12,7 +13,6 @@ apps: config: - ./buttercup_app_for_splunk/*.conf splunkbase-apps: - # Splunkbase apps Splunk Add-on for Amazon Web Services: version: 7.9.0 Cisco Networks Add-on for Splunk Enterprise: diff --git a/environments/uat/ses/deployment.yml b/environments/uat/ses/deployment.yml index 1d8d38f..2b9d76e 100644 --- a/environments/uat/ses/deployment.yml +++ b/environments/uat/ses/deployment.yml @@ -1,5 +1,6 @@ target: url: https://staging.admin.splunk.com/scv-shw-d037e758abafa2 + experience: victoria apps: Splunk_TA_app1: s3-bucket: splunk-apps-deployment @@ -7,7 +8,7 @@ apps: config: - ./Splunk_TA_app1/*.conf splunkbase-apps: - TA for Tebable: + Tenable Add-On for Splunk: version: 7.0.0 - TA for MS Azure: + Splunk Add on for Microsoft Azure: version: 4.2.0 From 2b21c8547f7f4da4cd06d6cc0146d94124a101e9 Mon Sep 17 00:00:00 2001 From: Erica Pescio Date: Tue, 21 Jan 2025 14:32:01 +0100 Subject: [PATCH 13/20] ci:added deps to workflow --- .github/workflows/deploy.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a3210a9..a65ec20 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -32,8 +32,7 @@ jobs: run: | python -m pip install --upgrade pip pip install --upgrade pyyaml - pip install boto3 - pip install requests + pip install boto3 requests schema - name: Deploy to ${{ matrix.environment.name }} continue-on-error: true env: From 89c53cf89a1f0bf22cbd581e31802785bcb3569a Mon Sep 17 00:00:00 2001 From: Erica Pescio Date: Tue, 21 Jan 2025 14:37:20 +0100 Subject: [PATCH 14/20] feat+refactor:added yml validation against schema --- deploy.py | 76 ++++++++---------- modules/apps_processing.py | 158 ++++++++++++++++++------------------- 2 files changed, 110 insertions(+), 124 deletions(-) diff --git a/deploy.py b/deploy.py index 24bfd2a..3cc23e1 100644 --- a/deploy.py +++ b/deploy.py @@ -14,44 +14,40 @@ def main(): # Initiate deployment report deployment_report = DeploymentReportGenerator() - # Initiate AppFilesProcessor object - app_processor = AppFilesProcessor() - # Initiate DeploymentParser object - deployment_parser = DeploymentParser() - ### 1. Validate data and retrieve all apps listed in deployment.yml from S3 ### - data, _, _ = deployment_parser.parse() - target_url = data["target"]["url"] - # Initiate AwsS3Connector object s3_connector = boto3.client("s3") + # Initiate DeploymentParser object + config = DeploymentParser() + # Initiate AppFilesProcessor object + app_processor = AppFilesProcessor(config) + # Initiate SplunkCloudConnector object + cloud_connector = SplunkCloudConnector( + config.url, config.cloud_experience + ) + # Check for private apps - if deployment_parser.has_private_apps(): - print("Found private apps in deployment.yml, starting deployment...") - # List all apps in yaml file and then their S3 buckets - apps = deployment_parser.get_private_apps() - s3_buckets = deployment_parser.get_s3_buckets() - app_directories = deployment_parser.get_app_directories() + if config.has_private_apps(): + print("Found private apps, starting deployment...") # Loop through all apps - for app, bucket, directory in zip(apps, s3_buckets, app_directories): - object_name = directory + for app in config.private_apps.keys(): + bucket = config.get_bucket(app) + app_path = config.get_app_path(app) file_name = f"{app}.tgz" # Donwload app from S3 try: - s3_connector.download_file(bucket, object_name, file_name) + s3_connector.download_file(bucket, app_path, file_name) except Exception as e: - raise Exception(f"Error downloading {object_name} from {bucket}: {e}") + raise Exception(f"Error downloading {app_path} from {bucket}: {e}") - ### 2. Upload_local_configuration ### - # Check if the configuration exists for the app + ### Upload_local_configuration ### + # Check whether the app needs specific configs for this env path = os.path.join(DEPLOYMENT_CONFIG_PATH, app) - if path: + if len(config.get_app_configs(app)) > 0: app_processor.unpack_merge_conf_and_meta_repack(app, path) else: - print(f"No configuration found for app {app}. Skipping.") + print(f"No configurations needed for app {app}. Skipping.") - ### 3. Validate app for Splunk Cloud ### - # Initiate SplunkCloudConnector object - cloud_connector = SplunkCloudConnector(target_url) + ### Validate app for Splunk Cloud ### appinspect_handler = cloud_connector.get_appinspect_handler() is_valid = appinspect_handler.validate(app) if not is_valid: @@ -61,9 +57,9 @@ def main(): deployment_report.add_data( app, ("distribution", "failed due to app validation error") ) - # App is valid + continue + ### App is valid: distribute it ### deployment_report.add_data(app, ("report", appinspect_handler.report)) - ### 4. If app is valid, distribute it ### dist_succeeded, dist_status = cloud_connector.distribute(app) if dist_succeeded: print(f"App {app} successfully distributed.\n") @@ -77,28 +73,24 @@ def main(): f"failed with status code: {dist_status}", ), ) - else: - print("No private apps found in deployment.yml, skipping...") + print("No private apps found, skipping...") - ### 5. Handle Splunkbase apps ### - if deployment_parser.has_splunkbase_apps(): - print("Found Splunkbase apps in deployment.yml, starting deployment...") - splunkbase_apps_dict = deployment_parser.get_splunkbase_apps() - for splunkbase_app in splunkbase_apps_dict: - app = splunkbase_apps_dict[splunkbase_app] - app_name = splunkbase_app - version = app["version"] - install_status = cloud_connector.install(app_name, version) - print(f"App {app_name} installation status: {install_status}") + ### Handle Splunkbase apps ### + if config.has_splunkbase_apps(): + print("Found Splunkbase apps, starting deployment...") + for splunkbase_app in config.splunkbase_apps.keys(): + version = config.get_version(app) + install_status = cloud_connector.install(app, version) + print(f"App {app} installation status: {install_status}") deployment_report.add_data( - app_name, + app, {"splunkbase_installation": install_status, "version": version}, ) else: - print("No Splunkbase apps found in deployment.yml, skipping...") + print("No Splunkbase apps found, skipping...") - ### 6. Save deployment report to json file ### + ### Save deployment report to json file ### deployment_report.generate_report() diff --git a/modules/apps_processing.py b/modules/apps_processing.py index e6653c4..240aed9 100644 --- a/modules/apps_processing.py +++ b/modules/apps_processing.py @@ -6,64 +6,64 @@ import tarfile import json from io import StringIO - +from schema import ( + Schema, + SchemaError, + Or, + Optional, + Regex +) + + +deployment_schema = Schema({ + "target": { + "url": str, + "experience": Or("classic", "victoria") + }, + "apps": { + Optional(str): { + "s3-bucket": str, + "source": str, + Optional("config"): [ + str + ] + } + }, + "splunkbase-apps": { + Optional(str): { + "version": Regex(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$") + } + } +}) class DeploymentParser: """Class for parsing the deployment configuration file.""" + private_apps: dict = {} + splunkbase_apps: dict = {} + target: dict = {} def __init__(self): + # Read and parse data if not "DEPLOYMENT_CONFIG_PATH" in os.environ: raise Exception( f"Error - Environment variable DEPLOYMENT_CONFIG_PATH does not exist." ) yml_path = os.path.join(os.getenv("DEPLOYMENT_CONFIG_PATH"), "deployment.yml") - # Read the file + try: with open(yml_path, "r") as file: - self.data = yaml.safe_load(file) + data = yaml.safe_load(file) + deployment_schema.validate(data) + + self.private_apps = data.get("apps", {}) + self.splunkbase_apps = data.get("splunkbase-apps", {}) + self.target = data.get("target", {}) except FileNotFoundError: raise Exception(f"File not found: {yml_path}") except yaml.YAMLError as e: raise Exception(f"Error parsing YAML file: {e}") - - def _validate_data(self) -> None: - """ - Validate the data in the YAML file. - - validate_data(data) -> None - """ - try: - data = self.data - except FileNotFoundError: - print(f"Error: The file '{self.yml_path}' was not found.") - except yaml.YAMLError as e: - print(f"Error parsing YAML file: {e}") - sys.exit(1) - - if "apps" not in data: - print("Error: The 'apps' key is missing in deploy.yml fime.") - sys.exit(1) - if "target" not in data: - print("Error: The 'target' key is missing in deploy.yml file.") - sys.exit(1) - if "url" not in data["target"]: - print("Error: The 'url' key is missing in the 'target' section.") - sys.exit(1) - if "splunkbase-apps" not in data: - print("Error: The 'splunkbase-apps' key is missing.") - sys.exit(1) - - def parse(self) -> tuple: - """ - Return the parsed data from the deployment configuration file. - - parse() -> (dict, dict, dict) - """ - self._validate_data() - private_apps_dict = self.data.get("apps", {}) - splunkbase_dict = self.data.get("splunkbase-apps", {}) - - return self.data, private_apps_dict, splunkbase_dict + except SchemaError as se: + raise Exception(f"Error validating {yml_path}: {se}") def has_private_apps(self) -> bool: """ @@ -71,8 +71,7 @@ def has_private_apps(self) -> bool: has_private_apps() -> bool """ - private_apps = self.parse()[1] - return True if private_apps else False + return True if self.private_apps else False def has_splunkbase_apps(self) -> bool: """ @@ -80,65 +79,60 @@ def has_splunkbase_apps(self) -> bool: has_splunkbase_apps() -> bool """ - splunkbase_apps = self.parse()[2] - return True if splunkbase_apps else False + return True if self.splunkbase_apps else False + + @property + def url(self) -> str: + """ + Return the targeted url from the deployment configuration. + """ + return self.target["url"] + + @property + def cloud_experience(self) -> str: + """ + Return the targeted platform cloud experience from the deployment configuration. + """ + return self.target["experience"] - def get_private_apps(self) -> list: + def get_bucket(self, app: str) -> str: """ - Return a dictionary of private apps from the deployment configuration. + Return the app S3 bucket from the deployment configuration. - get_apps() -> list + get_bucket(app) -> str """ - private_apps = self.parse()[1] - return private_apps.keys() + return self.private_apps[app]["s3-bucket"] - def get_s3_buckets(self) -> list: + def get_app_path(self, app: str) -> str: """ - Return a list of S3 buckets from the deployment configuration. + Return the app path from the deployment configuration. - s3_buckets() -> list + get_app_path(app) -> str """ - data = self.parse()[0] - apps = self.get_private_apps() - return [data["apps"][app]["s3-bucket"] for app in apps] + return self.private_apps[app]["source"] - def get_app_directories(self) -> list: + def get_app_configs(self, app: str) -> list: """ - Return a list of app directories from the deployment configuration. + Return a list of app configuration paths from the deployment configuration. - get_app_directories() -> list + get_app_configs(app) -> list """ - data = self.parse()[0] - apps = self.get_private_apps() - return [data["apps"][app]["source"] for app in apps] + return self.private_apps[app]["config"] - def get_splunkbase_apps(self) -> dict: + def get_version(self, app: str) -> str: """ - Return a dictionary of Splunkbase apps from the deployment configuration. + Return the Splunkbase app version from the deployment configuration. - get_splunkbase_apps() -> dict + get_version(app) -> str """ - splunkbase_apps = self.parse()[2] - return splunkbase_apps + return self.splunkbase_apps[app]["version"] class AppFilesProcessor: """Class for handling local app files and configurations.""" - def __init__(self): - if not "DEPLOYMENT_CONFIG_PATH" in os.environ: - raise Exception( - f"Error - Environment variable DEPLOYMENT_CONFIG_PATH does not exist." - ) - yml_path = os.path.join(os.getenv("DEPLOYMENT_CONFIG_PATH"), "deployment.yml") - # Read the file - try: - with open(yml_path, "r") as file: - self.data = yaml.safe_load(file) - except FileNotFoundError: - raise Exception(f"File not found: {yml_path}") - except yaml.YAMLError as e: - raise Exception(f"Error parsing YAML file: {e}") + def __init__(self, deployment_parser: DeploymentParser): + self.deployment_config = deployment_parser def _preprocess_empty_headers(self, file_path: str) -> list: """ From d864a9fc7c769ac05c9f2cda9fc9c3e69798a527 Mon Sep 17 00:00:00 2001 From: Erica Pescio Date: Tue, 21 Jan 2025 15:03:19 +0100 Subject: [PATCH 15/20] docs:edited README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 12fe308..160e839 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Splunk Apps Deployment Architecture -This is just an idea developed for Philips Electronics Nederland within the context of [JIRA ticket](https://splunk.atlassian.net/browse/FDSE-2571). To be extended and used at own risk. +This is just an idea developed within the context of [JIRA ticket](https://splunk.atlassian.net/browse/FDSE-2571). To be extended and used at own risk. Assumptions: * All apps are stored into a single GitHub repository From a57470ee80658f92bdebf1eb694a3a1a4cfc9a0e Mon Sep 17 00:00:00 2001 From: Erica Pescio Date: Tue, 21 Jan 2025 15:11:01 +0100 Subject: [PATCH 16/20] style:removed whitespace --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a65ec20..5ae4e82 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -44,7 +44,7 @@ jobs: DEPLOYMENT_CONFIG_PATH: "environments/${{ matrix.environment.name }}" run: | echo "Deploying to ${{ matrix.environment.name }} environment" - python -u deploy.py + python -u deploy.py - name: Upload deployment report as artifact uses: actions/upload-artifact@v3 with: From efecedb13218add718f067ac80df76f5c16b4fa5 Mon Sep 17 00:00:00 2001 From: Erica Pescio Date: Tue, 21 Jan 2025 15:11:19 +0100 Subject: [PATCH 17/20] docs:updated README --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 160e839..9249faf 100644 --- a/README.md +++ b/README.md @@ -50,10 +50,11 @@ This repository follows the same structure. Please navigate it to verify its con As mentioned, these deployment files specify the apps and configurations needed on each specific environment. Example: ```yml target: - url: + url: https://admin.splunk.com/{stack} experience: apps: # Private apps + # - Leave empty if target does not need private apps app1: s3-bucket: bucket-1 source: apps/app1.tgz @@ -63,7 +64,8 @@ apps: - ./app1/*.conf splunkbase-apps: # Splunkbase apps - cb-protection-app-for-splunk: + # - Leave empty if target does not need private apps + Cb Protection App for Splunk: version: 1.0.0 ``` From 060e14581d7b0856d5131c77fc5fd93f7d0cdab1 Mon Sep 17 00:00:00 2001 From: bdebek-splunk Date: Tue, 21 Jan 2025 15:20:08 +0100 Subject: [PATCH 18/20] fix: add break condition for successful app validation in AppInspectService to make execution faster --- modules/splunkcloud.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/splunkcloud.py b/modules/splunkcloud.py index 71f62bf..0d6ec42 100644 --- a/modules/splunkcloud.py +++ b/modules/splunkcloud.py @@ -97,6 +97,8 @@ def validate(self, app: str) -> bool: else: print(f"App {app} awaiting validation...") print(f"Current status: {response_json['status']}") + if response_json["status"] == "SUCCESS": + break time.sleep(10) if retries == max_retries: print(f"App {app} validation timed out.") From 4c0f6d768a772d6ba817a78045b3cfa457a9a8b9 Mon Sep 17 00:00:00 2001 From: bdebek-splunk Date: Tue, 21 Jan 2025 15:22:18 +0100 Subject: [PATCH 19/20] chore: remove unused AWS credentials from deployment workflows and script. fix: fixed variable naming from app to splunkbase_app --- .github/workflows/deploy.yml | 2 -- .github/workflows/manual_deploy.yml | 2 -- deploy.py | 15 +++++---------- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5ae4e82..2d82699 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -36,8 +36,6 @@ jobs: - name: Deploy to ${{ matrix.environment.name }} continue-on-error: true env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} SPLUNK_USERNAME: ${{ secrets.SPLUNK_USERNAME }} SPLUNK_PASSWORD: ${{ secrets.SPLUNK_PASSWORD }} SPLUNK_TOKEN: ${{ secrets[matrix.environment.token_key] }} diff --git a/.github/workflows/manual_deploy.yml b/.github/workflows/manual_deploy.yml index c31b766..6b9f71e 100644 --- a/.github/workflows/manual_deploy.yml +++ b/.github/workflows/manual_deploy.yml @@ -30,8 +30,6 @@ jobs: - name: Deploy to ${{ matrix.environment.name }} continue-on-error: true env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} SPLUNK_USERNAME: ${{ secrets.SPLUNK_USERNAME }} SPLUNK_PASSWORD: ${{ secrets.SPLUNK_PASSWORD }} SPLUNK_TOKEN: ${{ secrets[matrix.environment.token_key] }} diff --git a/deploy.py b/deploy.py index 3cc23e1..c886b38 100644 --- a/deploy.py +++ b/deploy.py @@ -5,9 +5,6 @@ from modules.apps_processing import AppFilesProcessor, DeploymentParser from modules.report_generator import DeploymentReportGenerator -AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID") -AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY") - DEPLOYMENT_CONFIG_PATH = os.getenv("DEPLOYMENT_CONFIG_PATH") @@ -21,9 +18,7 @@ def main(): # Initiate AppFilesProcessor object app_processor = AppFilesProcessor(config) # Initiate SplunkCloudConnector object - cloud_connector = SplunkCloudConnector( - config.url, config.cloud_experience - ) + cloud_connector = SplunkCloudConnector(config.url, config.cloud_experience) # Check for private apps if config.has_private_apps(): @@ -80,11 +75,11 @@ def main(): if config.has_splunkbase_apps(): print("Found Splunkbase apps, starting deployment...") for splunkbase_app in config.splunkbase_apps.keys(): - version = config.get_version(app) - install_status = cloud_connector.install(app, version) - print(f"App {app} installation status: {install_status}") + version = config.get_version(splunkbase_app) + install_status = cloud_connector.install(splunkbase_app, version) + print(f"App {splunkbase_app} installation status: {install_status}") deployment_report.add_data( - app, + splunkbase_app, {"splunkbase_installation": install_status, "version": version}, ) else: From 401e7d7c03173d8d13edeb3cc6b0339590004b6c Mon Sep 17 00:00:00 2001 From: bdebek-splunk Date: Tue, 21 Jan 2025 15:28:27 +0100 Subject: [PATCH 20/20] fix: revert deletion of AWS credentials in github workflow files --- .github/workflows/deploy.yml | 2 ++ .github/workflows/manual_deploy.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2d82699..5ae4e82 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -36,6 +36,8 @@ jobs: - name: Deploy to ${{ matrix.environment.name }} continue-on-error: true env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} SPLUNK_USERNAME: ${{ secrets.SPLUNK_USERNAME }} SPLUNK_PASSWORD: ${{ secrets.SPLUNK_PASSWORD }} SPLUNK_TOKEN: ${{ secrets[matrix.environment.token_key] }} diff --git a/.github/workflows/manual_deploy.yml b/.github/workflows/manual_deploy.yml index 6b9f71e..c31b766 100644 --- a/.github/workflows/manual_deploy.yml +++ b/.github/workflows/manual_deploy.yml @@ -30,6 +30,8 @@ jobs: - name: Deploy to ${{ matrix.environment.name }} continue-on-error: true env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} SPLUNK_USERNAME: ${{ secrets.SPLUNK_USERNAME }} SPLUNK_PASSWORD: ${{ secrets.SPLUNK_PASSWORD }} SPLUNK_TOKEN: ${{ secrets[matrix.environment.token_key] }}