Skip to content

Commit e027663

Browse files
kuntalkumarbasukuntalkumarbasu
authored
COREINF-7250:Enable schema evolution oxbow lambda (#38)
* enabling schema evolution for oxbow controlled tables * Enabling Glue sync * simple refactpr to take the glue create lambda its own function * Refactored the monitoring code * Fixed syntax * using var instead of local * moving to var instead of local * fixing monitoring syntax
1 parent 0900d03 commit e027663

File tree

5 files changed

+503
-301
lines changed

5 files changed

+503
-301
lines changed

‎glue_create.tf‎

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,273 @@
1+
2+
# glue-create lambda resource
3+
module "glue_create_athena_workgroup_bucket" {
4+
count = var.enable_glue_create ? 1 : 0
5+
6+
source = "terraform-aws-modules/s3-bucket/aws"
7+
version = "4.1.2"
8+
bucket = var.glue_create_config.athena_bucket_name
9+
block_public_acls = true
10+
block_public_policy = true
11+
ignore_public_acls = true
12+
restrict_public_buckets = true
13+
control_object_ownership = true
14+
object_ownership = "BucketOwnerEnforced"
15+
tags = var.tags
16+
versioning = {
17+
enabled = false
18+
}
19+
}
20+
21+
resource "aws_athena_workgroup" "glue_create" {
22+
count = var.enable_glue_create ? 1 : 0
23+
24+
name = var.glue_create_config.athena_workgroup_name
25+
tags = var.tags
26+
configuration {
27+
enforce_workgroup_configuration = true
28+
publish_cloudwatch_metrics_enabled = false
29+
30+
result_configuration {
31+
output_location = "s3://${module.glue_create_athena_workgroup_bucket[0].s3_bucket_id}/"
32+
}
33+
}
34+
depends_on = [module.glue_create_athena_workgroup_bucket]
35+
}
36+
37+
data "aws_iam_policy_document" "glue_create_sqs" {
38+
count = var.enable_glue_create ? 1 : 0
39+
40+
statement {
41+
effect = "Allow"
42+
principals {
43+
type = "*"
44+
identifiers = ["*"]
45+
}
46+
actions = ["sqs:SendMessage", "sqs:ReceiveMessage"]
47+
resources = ["arn:aws:sqs:*:*:${var.glue_create_config.sqs_queue_name}"]
48+
condition {
49+
test = "ArnEquals"
50+
variable = "aws:SourceArn"
51+
values = [var.glue_create_config.sns_topic_arn]
52+
}
53+
}
54+
}
55+
56+
data "aws_iam_policy_document" "glue_create_sqs_dl" {
57+
count = var.enable_glue_create ? 1 : 0
58+
59+
statement {
60+
effect = "Allow"
61+
principals {
62+
type = "AWS"
63+
identifiers = ["*"]
64+
}
65+
actions = ["sqs:SendMessage", "sqs:ReceiveMessage"]
66+
resources = ["arn:aws:sqs:*:*:${var.glue_create_config.sqs_queue_name_dl}"]
67+
condition {
68+
test = "ForAllValues:StringEquals"
69+
variable = "aws:SourceArn"
70+
values = ["arn:aws:sqs:*:*:${var.glue_create_config.sqs_queue_name}"]
71+
}
72+
}
73+
}
74+
75+
resource "aws_sqs_queue" "glue_create" {
76+
count = var.enable_glue_create ? 1 : 0
77+
78+
name = var.glue_create_config.sqs_queue_name
79+
policy = data.aws_iam_policy_document.glue_create_sqs[0].json
80+
visibility_timeout_seconds = var.sqs_visibility_timeout_seconds
81+
delay_seconds = var.sqs_delay_seconds
82+
redrive_policy = jsonencode({
83+
deadLetterTargetArn = aws_sqs_queue.glue_create_dl[0].arn
84+
maxReceiveCount = var.sqs_redrive_policy_maxReceiveCount
85+
})
86+
tags = var.tags
87+
}
88+
89+
resource "aws_sqs_queue" "glue_create_dl" {
90+
count = var.enable_glue_create ? 1 : 0
91+
92+
name = var.glue_create_config.sqs_queue_name_dl
93+
policy = data.aws_iam_policy_document.glue_create_sqs_dl[0].json
94+
tags = var.tags
95+
}
96+
97+
resource "aws_sqs_queue_redrive_allow_policy" "terraform_queue_redrive_allow_policy" {
98+
count = var.enable_glue_create ? 1 : 0
99+
100+
queue_url = aws_sqs_queue.glue_create_dl[0].id
101+
redrive_allow_policy = jsonencode({
102+
redrivePermission = "byQueue",
103+
sourceQueueArns = [aws_sqs_queue.glue_create[0].arn]
104+
})
105+
}
106+
107+
resource "aws_sns_topic_subscription" "glue_create_sns_sub" {
108+
count = var.enable_glue_create ? 1 : 0
109+
110+
topic_arn = var.glue_create_config.sns_topic_arn
111+
protocol = "sqs"
112+
endpoint = aws_sqs_queue.glue_create[0].arn
113+
}
114+
115+
data "aws_iam_policy_document" "glue_create_assume" {
116+
count = var.enable_glue_create ? 1 : 0
117+
118+
statement {
119+
effect = "Allow"
120+
principals {
121+
type = "Service"
122+
identifiers = ["lambda.amazonaws.com"]
123+
}
124+
actions = [
125+
"sts:AssumeRole",
126+
]
127+
}
128+
}
129+
130+
data "aws_iam_policy_document" "glue_create" {
131+
count = var.enable_glue_create ? 1 : 0
132+
133+
statement {
134+
sid = "AthenaWorkgroupAthenaRW"
135+
actions = [
136+
"athena:StartQueryExecution",
137+
"athena:GetQueryResults",
138+
"athena:GetWorkGroup",
139+
"athena:StopQueryExecution",
140+
"athena:GetQueryExecution",
141+
]
142+
resources = [
143+
aws_athena_workgroup.glue_create[0].arn
144+
]
145+
effect = "Allow"
146+
}
147+
statement {
148+
sid = "AthenaWorkgroupS3RW"
149+
effect = "Allow"
150+
actions = [
151+
"s3:PutObject",
152+
"s3:GetObject",
153+
"s3:AbortMultipartUpload",
154+
"s3:GetBucketLocation"
155+
]
156+
resources = [
157+
"${module.glue_create_athena_workgroup_bucket[0].s3_bucket_arn}/*",
158+
module.glue_create_athena_workgroup_bucket[0].s3_bucket_arn
159+
]
160+
}
161+
statement {
162+
sid = "AthenaWorkgroupList1"
163+
effect = "Allow"
164+
actions = ["athena:ListWorkGroups"]
165+
resources = ["*"]
166+
}
167+
statement {
168+
sid = "GlueAllowTables"
169+
effect = "Allow"
170+
actions = [
171+
"glue:GetTable",
172+
"glue:GetTables",
173+
"glue:GetPartitions",
174+
"glue:CreateTable",
175+
"glue:UpdateTable"
176+
]
177+
resources = [
178+
"arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:catalog",
179+
"arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:database/*",
180+
"arn:aws:glue:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:table/*"
181+
]
182+
}
183+
statement {
184+
sid = "GlueCatalogAllowDatabases"
185+
effect = "Allow"
186+
actions = [
187+
"glue:GetDatabase",
188+
"glue:GetDatabases",
189+
"glue:CreateDatabase"
190+
]
191+
resources = [
192+
"*"
193+
]
194+
}
195+
statement {
196+
sid = "TableExtLocS3RO"
197+
effect = "Allow"
198+
actions = [
199+
"s3:GetObject",
200+
"s3:GetObjectTagging",
201+
"s3:GetObjectVersion",
202+
"s3:GetBucketLocation",
203+
"s3:ListBucket",
204+
"s3:ListBucketVersions"
205+
]
206+
resources = [
207+
var.warehouse_bucket_arn,
208+
"${var.warehouse_bucket_arn}/${var.s3_path}/*"
209+
]
210+
}
211+
statement {
212+
effect = "Allow"
213+
actions = ["sqs:*"]
214+
resources = [aws_sqs_queue.glue_create[0].arn]
215+
}
216+
statement {
217+
effect = "Allow"
218+
actions = [
219+
"logs:CreateLogGroup",
220+
"logs:CreateLogStream",
221+
"logs:PutLogEvents"
222+
]
223+
resources = ["*"]
224+
}
225+
}
226+
227+
resource "aws_iam_policy" "glue_create_managed" {
228+
count = var.enable_glue_create ? 1 : 0
229+
230+
name = var.glue_create_config.iam_policy_name
231+
description = "Glue create policy allows access to Athena and S3"
232+
policy = data.aws_iam_policy_document.glue_create[0].json
233+
tags = var.tags
234+
}
235+
236+
resource "aws_iam_role" "glue_create" {
237+
count = var.enable_glue_create ? 1 : 0
238+
239+
name = var.glue_create_config.iam_role_name
240+
assume_role_policy = data.aws_iam_policy_document.glue_create_assume[0].json
241+
managed_policy_arns = [aws_iam_policy.glue_create_managed[0].arn]
242+
tags = var.tags
243+
}
244+
245+
resource "aws_lambda_function" "glue_create_lambda" {
246+
count = var.enable_glue_create ? 1 : 0
247+
248+
description = "Greate tables in AWS Glue catalog based on the table prefix"
249+
s3_key = var.glue_create_config.lambda_s3_key
250+
s3_bucket = var.glue_create_config.lambda_s3_bucket
251+
function_name = var.glue_create_config.lambda_function_name
252+
role = aws_iam_role.glue_create[0].arn
253+
handler = "provided"
254+
runtime = "provided.al2"
255+
memory_size = 1024
256+
timeout = 120
257+
environment {
258+
variables = {
259+
RUST_LOG = var.rust_log_oxbow_debug_level
260+
ATHENA_WORKGROUP = var.glue_create_config.athena_workgroup_name
261+
ATHENA_DATA_SOURCE = var.glue_create_config.athena_data_source
262+
GLUE_PATH_REGEX = var.glue_create_config.path_regex
263+
UNWRAP_SNS_ENVELOPE = true
264+
}
265+
}
266+
}
267+
268+
resource "aws_lambda_event_source_mapping" "glue_create" {
269+
count = var.enable_glue_create ? 1 : 0
270+
271+
event_source_arn = aws_sqs_queue.glue_create[0].arn
272+
function_name = aws_lambda_function.glue_create_lambda[0].arn
273+
}

0 commit comments

Comments
 (0)