Skip to content

Commit 15270f3

Browse files
committed
+ Support for python CI/CD
1 parent 33ba50b commit 15270f3

File tree

14 files changed

+383
-95
lines changed

14 files changed

+383
-95
lines changed

infrastructure-cdk/bin/infra-app.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,11 @@
22
import * as cdk from 'aws-cdk-lib';
33
import {InfraPipelineStack} from '../lib/infra-pipeline-stack';
44
import {ApplicationPipelineStack} from "../lib/application-pipeline-stack";
5+
import { Aspects } from 'aws-cdk-lib';
6+
import { AwsSolutionsChecks } from 'cdk-nag';
57

68
const app = new cdk.App();
79
new ApplicationPipelineStack(app, 'ApplicationPipelineStack');
810
new InfraPipelineStack(app, 'InfraPipelineStack');
11+
12+
// Aspects.of(app).add(new AwsSolutionsChecks({ verbose: true }));

infrastructure-cdk/data-source-function/app.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@
44
kinesis = boto3.client("kinesis")
55

66
def lambda_handler(event, context):
7-
kinesis.put_record(StreamName=os.environ["STREAM_NAME"], Data=b'This is test message from the Lambda function', PartitionKey="default")
7+
kinesis.put_record(StreamName=os.environ["STREAM_NAME"], Data=b'{"message":"This is test message from the Lambda function"}', PartitionKey="default")

infrastructure-cdk/lib/application-pipeline-stack.ts

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@ import {Aws, CfnOutput, RemovalPolicy, Stack, StackProps} from 'aws-cdk-lib';
22
import {Construct} from 'constructs';
33
import {JavaBuildPipeline} from "./constructs/java-build-pipeline";
44
import {BlockPublicAccess, Bucket, BucketEncryption} from "aws-cdk-lib/aws-s3";
5-
import {APPLICATION_NAME, ASSET_BUCKET_EXPORT_NAME} from "./shared-vars";
5+
import {APPLICATION_NAME, ASSET_BUCKET_EXPORT_NAME, BUILD_FOR_RUNTIME} from "./shared-vars";
6+
import {ApplicationRuntime} from "./constructs/application-runtime";
7+
import {PythonBuildPipeline} from "./constructs/python-build-pipeline";
68

79
export class ApplicationPipelineStack extends Stack {
810
constructor(scope: Construct, id: string, props?: StackProps) {
@@ -17,12 +19,22 @@ export class ApplicationPipelineStack extends Stack {
1719
removalPolicy: RemovalPolicy.DESTROY
1820
});
1921

20-
const javaBuildPipeline = new JavaBuildPipeline(this, 'java-app', {
21-
appName: APPLICATION_NAME,
22-
deployBucket: artifactBucket,
23-
repositoryName: APPLICATION_NAME,
24-
projectRoot: APPLICATION_NAME
25-
});
22+
let buildPipeline;
23+
// @ts-ignore
24+
if (BUILD_FOR_RUNTIME == ApplicationRuntime.JAVA)
25+
buildPipeline = new JavaBuildPipeline(this, 'java-app', {
26+
appName: APPLICATION_NAME,
27+
deployBucket: artifactBucket,
28+
repositoryName: APPLICATION_NAME,
29+
projectRoot: APPLICATION_NAME
30+
});
31+
else
32+
buildPipeline = new PythonBuildPipeline(this, 'python-app', {
33+
appName: APPLICATION_NAME,
34+
deployBucket: artifactBucket,
35+
repositoryName: APPLICATION_NAME,
36+
projectRoot: APPLICATION_NAME
37+
});
2638

2739
new CfnOutput(this, 'ArtifactBucketName', {
2840
value: artifactBucket.bucketName,
@@ -36,7 +48,7 @@ export class ApplicationPipelineStack extends Stack {
3648
});
3749

3850
new CfnOutput(this, 'ApplicationCodePipelineLink', {
39-
value: "https://console.aws.amazon.com/codesuite/codepipeline/pipelines/" + javaBuildPipeline.pipeline.pipelineName + "/view?region=" + Aws.REGION,
51+
value: "https://console.aws.amazon.com/codesuite/codepipeline/pipelines/" + buildPipeline.pipeline.pipelineName + "/view?region=" + Aws.REGION,
4052
description: "Application AWS CodePipeline Link"
4153
});
4254

infrastructure-cdk/lib/application-stack.ts

Lines changed: 32 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,16 @@ import {APPLICATION_NAME, ASSET_BUCKET_EXPORT_NAME} from "./shared-vars";
55
import {Stream, StreamEncryption, StreamMode} from "aws-cdk-lib/aws-kinesis";
66
import {Code, Function, Runtime} from "aws-cdk-lib/aws-lambda";
77
import * as kda from "@aws-cdk/aws-kinesisanalytics-flink-alpha";
8+
import {ApplicationRuntime} from "./constructs/application-runtime";
9+
10+
interface ApplicationStackProps {
11+
runtime: ApplicationRuntime,
12+
jarfile?: string
13+
}
14+
815

916
export class ApplicationStack extends Stack {
10-
constructor(scope: Construct, id: string, props?: StackProps) {
17+
constructor(scope: Construct, id: string, appProps: ApplicationStackProps, props?: StackProps) {
1118
super(scope, id, props);
1219
const assetBucket = Bucket.fromBucketName(this, 'imported-asset-bucket', Fn.importValue(ASSET_BUCKET_EXPORT_NAME));
1320

@@ -28,18 +35,33 @@ export class ApplicationStack extends Stack {
2835

2936
stream.grantWrite(dataSourceFn);
3037

38+
let propertyGroups: any = {
39+
"KinesisReader": {
40+
"input.stream.name": stream.streamName,
41+
"aws.region": Aws.REGION,
42+
"flink.stream.initpos": "LATEST"
43+
}
44+
};
45+
46+
let binaryPath = "jars/" + APPLICATION_NAME + "-latest.jar";
47+
if (appProps.runtime == ApplicationRuntime.PYTHON) {
48+
binaryPath = "python-binaries/" + APPLICATION_NAME + "-latest.zip";
49+
propertyGroups["kinesis.analytics.flink.run.options"] = {
50+
"python": "app.py",
51+
"pyFiles": "dependencies"
52+
};
53+
if (appProps.jarfile) {
54+
propertyGroups["kinesis.analytics.flink.run.options"]["jarfile"] = appProps.jarfile;
55+
}
56+
}
57+
58+
3159
const application = new kda.Application(this, 'app', {
3260
applicationName: APPLICATION_NAME,
33-
code: kda.ApplicationCode.fromBucket(assetBucket, "jars/" + APPLICATION_NAME + "-latest.jar"),
61+
code: kda.ApplicationCode.fromBucket(assetBucket, binaryPath),
3462
runtime: kda.Runtime.FLINK_1_13,
35-
propertyGroups: {
36-
"KinesisReader": {
37-
"input.stream.name": stream.streamName,
38-
"aws.region": Aws.REGION,
39-
"flink.stream.initpos": "LATEST"
40-
},
41-
},
42-
snapshotsEnabled: false,
63+
propertyGroups: propertyGroups,
64+
snapshotsEnabled: false, // true for the real environment
4365
parallelismPerKpu: 1,
4466
removalPolicy: RemovalPolicy.DESTROY
4567
}
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
export enum ApplicationRuntime {
2+
JAVA,
3+
PYTHON
4+
}

infrastructure-cdk/lib/constructs/java-build-pipeline.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ export class JavaBuildPipeline extends Construct {
128128

129129
versionUpdateFn.addToRolePolicy(new PolicyStatement({
130130
actions: ["kinesisanalytics:DescribeApplication", "kinesisanalytics:UpdateApplication"],
131-
resources: ["arn:aws:kinesisanalytics:" + Aws.REGION + ":" + Aws.ACCOUNT_ID + ":application/*"]
131+
resources: ["arn:aws:kinesisanalytics:" + Aws.REGION + ":" + Aws.ACCOUNT_ID + ":application/" + props.appName]
132132
}));
133133
versionUpdateFn.addToRolePolicy(new PolicyStatement({
134134
resources: ["*"],
Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
import {Construct} from "constructs";
2+
import {BuildSpec, LinuxBuildImage, PipelineProject} from "aws-cdk-lib/aws-codebuild";
3+
import {IBucket} from "aws-cdk-lib/aws-s3";
4+
import {Artifact, Pipeline} from "aws-cdk-lib/aws-codepipeline";
5+
import {
6+
CodeBuildAction,
7+
LambdaInvokeAction,
8+
ManualApprovalAction,
9+
S3DeployAction,
10+
S3SourceAction
11+
} from "aws-cdk-lib/aws-codepipeline-actions";
12+
import {Code, Function, Runtime} from "aws-cdk-lib/aws-lambda";
13+
import {Aws, Duration} from "aws-cdk-lib";
14+
import {PolicyStatement} from "aws-cdk-lib/aws-iam";
15+
import {SOURCE_CODE_ZIP} from "../shared-vars";
16+
17+
18+
interface PythonBuildPipelineProps {
19+
appName: string
20+
repositoryName: string
21+
deployBucket: IBucket
22+
projectRoot?: string
23+
deployBucketBasePath?: string
24+
}
25+
26+
export class PythonBuildPipeline extends Construct {
27+
readonly pipeline: Pipeline
28+
29+
constructor(scope: Construct, id: string, props: PythonBuildPipelineProps) {
30+
super(scope, id);
31+
32+
let directory = ".";
33+
let s3BasePath = "python-binaries";
34+
if (props.projectRoot) {
35+
directory = props.projectRoot;
36+
}
37+
if (props.deployBucketBasePath) {
38+
s3BasePath = props.deployBucketBasePath
39+
}
40+
41+
const sourceAsset = new Artifact();
42+
const defaultBuildSpec = BuildSpec.fromObject({
43+
version: '0.2',
44+
phases: {
45+
install: {
46+
"runtime-versions": {
47+
"python": "3.x"
48+
}
49+
},
50+
build: {
51+
commands: [
52+
`cd ${directory}`,
53+
'mkdir dependencies',
54+
'pip3 install -r requirements.txt -t dependencies',
55+
`mkdir -p ${s3BasePath}`,
56+
`zip -r ${s3BasePath}/${props.appName}-latest.zip *.py lib/* dependencies`
57+
]
58+
}
59+
},
60+
artifacts: {
61+
files: [
62+
`${s3BasePath}/${props.appName}.zip`
63+
],
64+
'discard-paths': true,
65+
'base-directory': directory
66+
}
67+
});
68+
69+
const project = new PipelineProject(this, 'Pipeline', {
70+
environment: {
71+
buildImage: LinuxBuildImage.STANDARD_5_0
72+
},
73+
buildSpec: defaultBuildSpec
74+
});
75+
76+
const buildOutput = new Artifact();
77+
78+
this.pipeline = new Pipeline(this, 'CodePipeline', {
79+
stages: [
80+
// In real world use code snippet like below to work with repository
81+
//
82+
// {
83+
// stageName: "source", actions: [new GitHubSourceAction({
84+
// actionName: "CodeCheckout",
85+
// oauthToken: SecretValue.secretsManager("github-secret"),
86+
// output: sourceAsset,
87+
// owner: "some-owner",
88+
// repo: props.repositoryName
89+
// })]
90+
// },
91+
{
92+
stageName: "source", actions: [new S3SourceAction({
93+
output: sourceAsset,
94+
actionName: "Checkout",
95+
bucket: props.deployBucket,
96+
bucketKey: SOURCE_CODE_ZIP
97+
})]
98+
},
99+
{
100+
stageName: "build", actions: [new CodeBuildAction({
101+
input: sourceAsset, actionName: "CodeBuild", project: project, outputs: [buildOutput]
102+
})]
103+
}, {
104+
stageName: "saveArtifact", actions: [new S3DeployAction({
105+
bucket: props.deployBucket,
106+
actionName: "SaveArtifact",
107+
input: buildOutput,
108+
extract: true
109+
})]
110+
}, {
111+
stageName: "approval",
112+
actions: [new ManualApprovalAction({
113+
actionName: "Manual"
114+
})]
115+
}]
116+
});
117+
118+
const versionUpdateFn = new Function(this, 'version-update-fn', {
119+
code: Code.fromAsset('flink-app-redeploy-hook'),
120+
handler: "app.lambda_handler",
121+
runtime: Runtime.PYTHON_3_9,
122+
environment: {
123+
ASSET_BUCKET_ARN: props.deployBucket.bucketArn,
124+
FILE_KEY: s3BasePath + "/" + props.appName + "-latest.zip",
125+
APP_NAME: props.appName
126+
},
127+
timeout: Duration.minutes(1)
128+
});
129+
130+
versionUpdateFn.addToRolePolicy(new PolicyStatement({
131+
actions: ["kinesisanalytics:DescribeApplication", "kinesisanalytics:UpdateApplication"],
132+
resources: ["arn:aws:kinesisanalytics:" + Aws.REGION + ":" + Aws.ACCOUNT_ID + ":application/" + props.appName]
133+
}));
134+
versionUpdateFn.addToRolePolicy(new PolicyStatement({
135+
resources: ["*"],
136+
actions: ["codepipeline:PutJobSuccessResult", "codepipeline:PutJobFailureResult"]
137+
}));
138+
139+
this.pipeline.addStage({
140+
stageName: "deploy", actions: [new LambdaInvokeAction({
141+
actionName: "Deploy",
142+
lambda: versionUpdateFn
143+
})]
144+
});
145+
146+
}
147+
}
Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,23 @@
11
import {Stage, StageProps} from "aws-cdk-lib";
22
import {Construct} from "constructs";
33
import {ApplicationStack} from "./application-stack";
4+
import {ApplicationRuntime} from "./constructs/application-runtime";
5+
import {BUILD_FOR_RUNTIME} from "./shared-vars";
46

57

68
export class RealtimeApplication extends Stage {
79
constructor(scope: Construct, id: string, props?: StageProps) {
810
super(scope, id, props);
911

10-
new ApplicationStack(this, 'ApplicationStack');
12+
// @ts-ignore
13+
if (BUILD_FOR_RUNTIME == ApplicationRuntime.JAVA)
14+
new ApplicationStack(this, 'ApplicationStack', {
15+
runtime: ApplicationRuntime.JAVA
16+
});
17+
else
18+
// For python
19+
new ApplicationStack(this, 'ApplicationStack', {
20+
runtime: ApplicationRuntime.PYTHON
21+
});
1122
}
1223
}
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
import {ApplicationRuntime} from "./constructs/application-runtime";
2+
13
export const APPLICATION_NAME = "kinesis-analytics-application"
4+
export const BUILD_FOR_RUNTIME = ApplicationRuntime.PYTHON
25
export const SOURCE_CODE_ZIP = "automate-deployment-and-version-update-of-kda-application.zip"
36
export const ASSET_BUCKET_EXPORT_NAME = "Blog::Artifact::BucketName"

0 commit comments

Comments
 (0)