Extending readme and cleanup of some code

Author: YYC

.gitignore

@@ -127,3 +127,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# Zip files
+.zip

README.md

@@ -1,2 +1,32 @@
 # lambda-appflow-ga
-Lambda function in python to transform AWS Appflow Google Analytics data
+Lambda function in python to transform AWS Appflow Google Analytics data to tabular data.
+
+## Description
+This lambda is to be used in combination with Appflow for google analytics.
+The lambda will trigger every time a file is written to S3 and will convert the RAW JSON to a table
+structure that can be written to Postgresql.
+Only google analytics has been tested, test compatibility with others if needed.
+
+This solution is part of a larger blog on medium, check the complete story there (`https://medium.com/@yvescallaert`)
+
+
+## Requirements
+You will need the following to contribute to this project
+* Python 3.7
+* Pip install of requirements.txt
+* Pip install boto3
+
+## Deploying
+A deployment to AWS can be done using the following command from the root of the directory
+`./packager.sh -n test_appflow -z test_appflow -p webanalytics -r eu-central-1`
+
+Note that the script will look for an IAM role called `lambda-cli-role`, if this role is not present you will need to
+create it. The role should contain the following policies:
+* AWSLambdaBasicExecutionRole
+* AWSLambdaVPCAccessExecutionRole
+* SecretsManagerReadWrite (optional if you don't want to store your secrets in AWS Secrets Manager)
+* Custom Policy that gives full access to the S3 destionation folder used in AppFlow
+
+## Note
+Please note that this script has been tested with a max of 2 metrics and 5 dimensions. The total result during testing
+was never bigger than 2K records. If your use case is over 2K records you might need to alter parts of this script.

appflow_ga.py

@@ -12,6 +12,7 @@
         logger.removeHandler(handler)
 logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)
 
+
 class PostgressDB():
     def __init__(self, username, password, port, host, database):
         self.user_name = username
@@ -45,7 +46,7 @@ def process_json_file(jsonf):
     """
     logger.info("Starting conversion JSON format to table format.")
     logger.info("Detecting {} valid JSON structures in the objects".format(str(len(jsonf))))
-    #JsonF is a list but the cols and metrics will always be the same across multiple jsons for 1 file
+    # JsonF is a list but the cols and metrics will always be the same across multiple jsons for 1 file
     cols = []
     try:
         cols = [r for r in jsonf[0]['reports'][0]['columnHeader']['dimensions']]
@@ -57,15 +58,14 @@ def process_json_file(jsonf):
     except:
         logger.warning("No metrics specified.")
 
-
     pd_result = None
 
     for list_index in range(len(jsonf)):
         data_rows = [r for r in jsonf[list_index]['reports'][0]['data']['rows']]
         dim_result_dict = {}
 
         for row in data_rows:
-            #if there are dimensions, extract the dimension data and add values per key
+            # if there are dimensions, extract the dimension data and add values per key
             for i in range(len(cols)):
                 if cols[i] in dim_result_dict.keys():
                     data_list = dim_result_dict[cols[i]]
@@ -82,25 +82,26 @@ def process_json_file(jsonf):
                     dim_result_dict.update({metrics[i]: data_list})
                 else:
                     dim_result_dict[metrics[i]] = [row['metrics'][0]['values'][i]]
-        #Create dataframe for the first JSON object otherwise append to existing
+        # Create dataframe for the first JSON object otherwise append to existing
         if list_index == 0:
             pd_result = pd.DataFrame.from_dict(dim_result_dict)
         else:
             pd_result = pd_result.append(pd.DataFrame.from_dict(dim_result_dict))
         logger.info("Finished conversion JSON format to table format.")
     return pd_result
 
+
 def lambda_handler(event, context):
     logger.info("Starting appflow conversion")
     bucket_name = event['Records'][0]['s3']['bucket']['name']
     object_key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'])
     s3_client = boto3.client('s3')
 
-    logger.info("Processing bucket {}, filename {}".format(bucket_name,object_key))
+    logger.info("Processing bucket {}, filename {}".format(bucket_name, object_key))
 
     raw_object = s3_client.get_object(Bucket=bucket_name, Key=object_key)
     raw_data = json.loads('[' + raw_object['Body'].read().decode('utf-8').replace('}\n{', '},{') + ']')
-    #Raw data is always a list of JSON objects
+    # Raw data is always a list of JSON objects
     pd_result = process_json_file(raw_data)
 
     db = PostgressDB(username=os.getenv("DB_USERNAME"),

packager.sh

@@ -7,6 +7,9 @@ nFlag=0
 zFlag=0
 
 CreateZipFile(){
+rm -rf package
+mkdir -p package
+pip install -r requirements.txt --target package/
 cd package
 zip -r9 ../${ZIP_NAME}.zip .
 cd ../
@@ -15,7 +18,7 @@ zip -g ${ZIP_NAME}.zip appflow_ga.py
 
 
 UsageMessage(){
-echo "usage : packager -p <aws profile> -r <aws_region> -n <name of the function> -z <zipname> -h <dbhost> -i <GOOGLE_SPREADSHEET_ID> -t <GOOGLE_SPREADSHEET_TAB_PAGE> -d <DB_TEMP_TABLE>"
+echo "usage : packager -p <aws profile> -r <aws_region> -n <name of the function> -z <zipname>"
 }
 
 while getopts "p:r:n:z:" option
@@ -48,7 +51,6 @@ fi
 CreateZipFile
 
 
-
 function_name_cli=`aws lambda get-function --function-name ${FUNCTION_NAME} --profile ${AWS_PROFILE} --region=${AWS_REGION} | jq -r '.Configuration | .FunctionName'`
 
 if [ ${function_name_cli} == ${FUNCTION_NAME} ]; then