Initial Commit

Author: anandshah123

.gitignore

@@ -0,0 +1,2 @@
+.idea
+**/node_modules

infrastructure-cdk/bin/infra-app.ts

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+import * as cdk from 'aws-cdk-lib';
+import {InfraPipelineStack} from '../lib/infra-pipeline-stack';
+import {ApplicationPipelineStack} from "../lib/application-pipeline-stack";
+
+const app = new cdk.App();
+new ApplicationPipelineStack(app, 'ApplicationPipelineStack');
+new InfraPipelineStack(app, 'InfraPipelineStack');

infrastructure-cdk/cdk.json

@@ -0,0 +1,42 @@
+{
+  "app": "npx ts-node --prefer-ts-exts bin/infra-app.ts",
+  "watch": {
+    "include": [
+      "**"
+    ],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "**/*.d.ts",
+      "**/*.js",
+      "tsconfig.json",
+      "package*.json",
+      "yarn.lock",
+      "node_modules",
+      "test"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true,
+    "@aws-cdk/core:stackRelativeExports": true,
+    "@aws-cdk/aws-rds:lowercaseDbIdentifier": true,
+    "@aws-cdk/aws-lambda:recognizeVersionProps": true,
+    "@aws-cdk/aws-lambda:recognizeLayerVersion": true,
+    "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true,
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
+    "@aws-cdk/core:checkSecretUsage": true,
+    "@aws-cdk/aws-iam:minimizePolicies": true,
+    "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true,
+    "@aws-cdk/core:validateSnapshotRemovalPolicy": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
+    "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true,
+    "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true,
+    "@aws-cdk/aws-apigateway:disableCloudWatchRole": true,
+    "@aws-cdk/core:enablePartitionLiterals": true,
+    "@aws-cdk/core:target-partitions": [
+      "aws",
+      "aws-cn"
+    ]
+  }
+}

infrastructure-cdk/jest.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+  testEnvironment: 'node',
+  roots: ['<rootDir>/test'],
+  testMatch: ['**/*.test.ts'],
+  transform: {
+    '^.+\\.tsx?$': 'ts-jest'
+  }
+};

infrastructure-cdk/lib/application-pipeline-stack.ts

@@ -0,0 +1,33 @@
+import {Aws, CfnOutput, Stack, StackProps} from 'aws-cdk-lib';
+import {Construct} from 'constructs';
+import {JavaBuildPipeline} from "./constructs/java-build-pipeline";
+import {BlockPublicAccess, Bucket, BucketEncryption} from "aws-cdk-lib/aws-s3";
+import {ASSET_BUCKET_EXPORT_NAME, REPOSITORY_FILE_PATH} from "./shared-vars";
+
+export class ApplicationPipelineStack extends Stack {
+    constructor(scope: Construct, id: string, props?: StackProps) {
+        super(scope, id, props);
+
+        const artefactBucket = new Bucket(this, 'ArtefactBucket', {
+            blockPublicAccess: BlockPublicAccess.BLOCK_ALL,
+            enforceSSL: true,
+            encryption: BucketEncryption.S3_MANAGED
+        });
+
+        new JavaBuildPipeline(this, 'java-app', {
+            deployBucket: artefactBucket, repositoryName: REPOSITORY_FILE_PATH
+        });
+
+        new CfnOutput(this, 'AssetBucketName', {
+            value: artefactBucket.bucketName,
+            description: "Artefact Bucket name storing application binaries",
+            exportName: ASSET_BUCKET_EXPORT_NAME
+        });
+
+        new CfnOutput(this, 'AssetBucketLink', {
+            value: "https://s3.console.aws.amazon.com/s3/buckets/" + artefactBucket.bucketName + "?region=" + Aws.REGION + "&tab=objects",
+            description: "Artefact Bucket Link"
+        });
+
+    }
+}

infrastructure-cdk/lib/constructs/java-build-pipeline.ts

@@ -0,0 +1,114 @@
+import {Construct} from "constructs";
+import {BuildSpec, LinuxBuildImage, PipelineProject} from "aws-cdk-lib/aws-codebuild";
+import {IBucket} from "aws-cdk-lib/aws-s3";
+import {Artifact, Pipeline} from "aws-cdk-lib/aws-codepipeline";
+import {
+    CodeBuildAction,
+    LambdaInvokeAction,
+    ManualApprovalAction,
+    S3DeployAction
+} from "aws-cdk-lib/aws-codepipeline-actions";
+import {IFunction} from "aws-cdk-lib/aws-lambda/lib/function-base";
+
+
+interface JavaBuildPipelineProps {
+    repositoryName: string
+    deployBucket: IBucket
+    projectRoot?: string
+    deployBucketBasePath?: string
+    postActionLambda?: IFunction
+}
+
+export class JavaBuildPipeline extends Construct {
+    constructor(scope: Construct, id: string, props: JavaBuildPipelineProps) {
+        super(scope, id);
+
+        let directory = ".";
+        let s3BasePath = "jars";
+        if (props.projectRoot) {
+            directory = props.projectRoot;
+        }
+        if (props.deployBucketBasePath) {
+            s3BasePath = props.deployBucketBasePath
+        }
+
+        const sourceAsset = new Artifact();
+        const defaultBuildSpec = BuildSpec.fromObject({
+            version: '0.2',
+            phases: {
+                install: {
+                    "runtime-versions": {
+                        "java": "corretto11"
+                    }
+                },
+                build: {
+                    commands: [
+                        `curl ${props.repositoryName} --output app.zip`, // Download zip directly from Github
+                        'unzip app.zip',
+                        `cd ${directory}`,
+                        'mvn clean package -B',
+                        `mkdir -p ${s3BasePath}`,
+                        `cp target/*.jar ${s3BasePath}/`
+                    ]
+                }
+            },
+            artifacts: {
+                files: [
+                    `${s3BasePath}/*.jar`
+                ],
+                'discard-paths': false,
+                'base-directory': directory
+            }
+        });
+
+        const project = new PipelineProject(this, 'Pipeline', {
+            environment: {
+                buildImage: LinuxBuildImage.STANDARD_5_0
+            },
+            buildSpec: defaultBuildSpec
+        });
+
+        const buildOutput = new Artifact();
+
+        const pipeline = new Pipeline(this, 'CodePipeline', {
+            stages: [
+                // In real world use code snippet like below to work with repository
+                //
+                // {
+                //     stageName: "source", actions: [new GitHubSourceAction({
+                //         actionName: "CodeCheckout",
+                //         oauthToken: SecretValue.secretsManager("github-secret"),
+                //         output: sourceAsset,
+                //         owner: "some-owner",
+                //         repo: props.repositoryName
+                //     })]
+                // },
+                {
+                    stageName: "build", actions: [new CodeBuildAction({
+                        actionName: "CodeBuild", input: sourceAsset, project: project, outputs: [buildOutput]
+                    })]
+                }, {
+                    stageName: "saveArtefact", actions: [new S3DeployAction({
+                        bucket: props.deployBucket,
+                        actionName: "SaveArtefact",
+                        input: buildOutput,
+                        extract: true
+                    })]
+                }, {
+                    stageName: "approval",
+                    actions: [new ManualApprovalAction({
+                        actionName: "Manual"
+                    })]
+                }]
+        });
+
+        if (props.postActionLambda) {
+            pipeline.addStage({
+                stageName: "deploy", actions: [new LambdaInvokeAction({
+                    actionName: "Deploy",
+                    lambda: props.postActionLambda
+                })]
+            });
+        }
+    }
+}

infrastructure-cdk/lib/infra-pipeline-stack.ts

@@ -0,0 +1,29 @@
+import {Stack, StackProps} from 'aws-cdk-lib';
+import {Construct} from 'constructs';
+import {CodePipeline, ShellStep} from "aws-cdk-lib/pipelines";
+import {RealtimeApplication} from "./real-time-application";
+import {REPOSITORY_FILE_PATH} from "./shared-vars";
+
+export class InfraPipelineStack extends Stack {
+    constructor(scope: Construct, id: string, props?: StackProps) {
+        super(scope, id, props);
+
+
+        const pipeline = new CodePipeline(this, 'Pipeline', {
+            selfMutation: false,
+            synth: new ShellStep('Synth', {
+                commands: [
+                    "curl " + REPOSITORY_FILE_PATH + " --output app.zip",
+                    "unzip app.zip",
+                    "cd infrastructure-cdk",
+                    "npm ci",
+                    "npm run build",
+                    "npm cdk synth"
+                ],
+                primaryOutputDirectory: 'infrastructure-cdk/cdk.out'
+            })
+        });
+
+        pipeline.addStage(new RealtimeApplication(this, 'app'));
+    }
+}

infrastructure-cdk/lib/real-time-application.ts

@@ -0,0 +1,56 @@
+import {Aws, Fn, Stage, StageProps} from "aws-cdk-lib";
+import {Construct} from "constructs";
+import {Code, Function, Runtime} from "aws-cdk-lib/aws-lambda";
+import {Stream, StreamEncryption, StreamMode} from "aws-cdk-lib/aws-kinesis";
+import * as kda from "@aws-cdk/aws-kinesisanalytics-flink-alpha";
+import {APPLICATION_NAME, ASSET_BUCKET_EXPORT_NAME} from "./shared-vars";
+import {Bucket} from "aws-cdk-lib/aws-s3";
+
+
+export class RealtimeApplication extends Stage {
+    constructor(scope: Construct, id: string, props?: StageProps) {
+        super(scope, id, props);
+
+        const assetBucket = Bucket.fromBucketName(this, 'imported-asset-bucket', Fn.importValue(ASSET_BUCKET_EXPORT_NAME));
+
+        const stream = new Stream(this, 'raw', {
+            streamMode: StreamMode.PROVISIONED,
+            shardCount: 1,
+            encryption: StreamEncryption.MANAGED
+        });
+
+        const dataSourceFn = new Function(this, 'data-source', {
+            code: Code.fromInline(
+                "import boto3\n" +
+                "import os\n\n" +
+                "kinesis = boto3.client('kinesis')\n\n" +
+                "def lambda_handler(event, context):\n" +
+                "   kinesis.put_record(StreamName=os.environ['STREAM_NAME']), Data=b'{\"message\":\"This is the test message from the lambda\"}', PartitionKey='default')\n"
+            ),
+            handler: "index.lambda_handler",
+            runtime: Runtime.PYTHON_3_9,
+            environment: {
+                STREAM_NAME: stream.streamName
+            }
+        });
+
+        stream.grantWrite(dataSourceFn);
+
+        const application = new kda.Application(this, 'app', {
+                code: kda.ApplicationCode.fromBucket(assetBucket, "jars/" + APPLICATION_NAME + "-1.0.0.jar"),
+                runtime: kda.Runtime.FLINK_1_13,
+                propertyGroups: {
+                    "KinesisReader": {
+                        "input.stream.name": stream.streamName,
+                        "aws.region": Aws.REGION,
+                        "flink.stream.initpos": "LATEST"
+                    },
+                },
+                snapshotsEnabled: false,
+                parallelismPerKpu: 1
+            }
+        );
+
+        stream.grantRead(application);
+    }
+}

infrastructure-cdk/lib/shared-vars.ts

@@ -0,0 +1,3 @@
+export const REPOSITORY_FILE_PATH = "aws-samples/automate-deployment-and-version-update-of-kda-application"
+export const APPLICATION_NAME = "kinesis-analytics-application"
+export const ASSET_BUCKET_EXPORT_NAME = "Blog::Artefact::BucketName"