Skip to content

Commit e6ff865

Browse files
adisamantadisamant
committed
1\ Now tracks all regions of the global cluster in the dynamodb entry
2\ Now removes dynamodb entry from all regions when an unplanned failover happens
1 parent a848fe9 commit e6ff865

File tree

2 files changed

+130
-27
lines changed

2 files changed

+130
-27
lines changed

‎create_managed_endpoint.py‎

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import argparse
66
import boto3
77
from botocore.exceptions import ClientError
8+
from boto3.dynamodb.types import TypeSerializer,TypeDeserializer
89
import re
910
import sys
1011
import datetime
@@ -198,8 +199,25 @@ def hosted_zone_id(hzonename):
198199
print(e)
199200
raise
200201

201-
def make_ddb_entry(cluname,hzid,recordname,region):
202-
#Make cluster, hostedzone entries in the dynamodb table
202+
#serializes python dict obj to dynamodb for insertion
203+
def srl_ddb(python_obj: dict) -> dict:
204+
serializer = TypeSerializer()
205+
return {
206+
k: serializer.serialize(v)
207+
for k, v in python_obj.items()
208+
}
209+
210+
#deserializes to dynamodb map data to python dict obj for use
211+
def dsrl_ddb(dynamo_obj: dict) -> dict:
212+
deserializer = TypeDeserializer()
213+
return {
214+
k: deserializer.deserialize(v)
215+
for k, v in dynamo_obj.items()
216+
}
217+
218+
219+
def make_ddb_entry(cluname,hzid,recordname,region,allregions):
220+
#Make cluster, hostedzone and region entries in the dynamodb table
203221
try:
204222
ddbclient=boto3.client('dynamodb',region_name = region)
205223

@@ -217,6 +235,9 @@ def make_ddb_entry(cluname,hzid,recordname,region):
217235
},
218236
"region": {
219237
"S": region
238+
},
239+
"allregions": {
240+
"M": allregions
220241
}
221242
}
222243
)
@@ -347,11 +368,26 @@ def main():
347368
gdbclient =boto3.client("rds",region_name = region)
348369
response=gdbclient.describe_global_clusters(GlobalClusterIdentifier = gdbcluname)
349370

350-
# define boto client globally, since they get used in functions.
371+
# define dns client globally, since they get used in functions outside of the main function.
351372

352373
global dnsclient
353374
dnsclient = boto3.client("route53", region_name = region)
354375

376+
#find all regions for this global database, and populate in allregion array
377+
378+
allregions = {}
379+
380+
for i in response['GlobalClusters'][0]['GlobalClusterMembers']:
381+
resourcename = i['DBClusterArn']
382+
resourcename = resourcename.split(':')
383+
regioname = resourcename[3] #region name is in the 3rd postion
384+
cluname = resourcename[6] #clustername is in the 6th position
385+
allregions[cluname]=regioname
386+
387+
#serialize the data
388+
gdbobj = srl_ddb(allregions)
389+
390+
355391
# Loop thorugh each regional cluster member for the provided global cluster
356392
for i in response['GlobalClusters'][0]['GlobalClusterMembers']:
357393
resourcename = i['DBClusterArn'] #This is the ARN
@@ -391,7 +427,7 @@ def main():
391427
print ("CNAME",recordname,"doesn't exist. Adding CNAME record..")
392428
create_hosted_zone_record(hostedzonename,recordname,recordvalue) # create a cname record in the hosted zone for the writer endpoint
393429
hzid = hosted_zone_id(hostedzonename) #get hosted zone id
394-
make_ddb_entry(cluname,hzid,recordname,regioname) # Add dynamodb entry
430+
make_ddb_entry(cluname,hzid,recordname,regioname,gdbobj) # Add dynamodb entry
395431
else:
396432
if (skipvpc == False):
397433
print ("Vpc",instancevpc," doesn't exist. Adding vpc..")
@@ -407,12 +443,12 @@ def main():
407443
print ("CNAME",recordname,"doesn't exist. Adding CNAME record for Primary region cluster", cluname)
408444
create_hosted_zone_record(hostedzonename,recordname,recordvalue) # create a cname record in the hosted zone for the writer endpoint
409445
hzid = hosted_zone_id(hostedzonename) #get hosted zone id
410-
make_ddb_entry(cluname,hzid,recordname,regioname) # Add dynamodb entry
446+
make_ddb_entry(cluname,hzid,recordname,regioname,gdbobj) # Add dynamodb entry
411447
else:
412448
print ("Hosted Zone doesn't exists. Creating ",hostedzonename)
413449
hzid = create_hosted_zone(hostedzonename,regioname,instancevpc)
414450

415-
make_ddb_entry(cluname,hzid,recordname,regioname) #Make ddb entry. This should only work from the calling region.
451+
make_ddb_entry(cluname,hzid,recordname,regioname,gdbobj) #Make ddb entry. This should only work from the calling region.
416452

417453
recordvalue = get_writer_endpoint(cluname) # Get writer endpoint for the cluster
418454
print ("Adding CNAME record ", recordname,"for for Primary region cluster ",cluname)
@@ -440,20 +476,20 @@ def main():
440476
print ("VPC ",instancevpc,"Already exists for secondary region cluster ", cluname)
441477

442478
hzid = hosted_zone_id(hostedzonename) #get hosted zone id
443-
make_ddb_entry(cluname,hzid,recordname,regioname) # Add dynamodb entry
479+
make_ddb_entry(cluname,hzid,recordname,regioname,gdbobj) # Add dynamodb entry
444480

445481
else:
446482
if (skipvpc == False):
447483
print ("VPC",instancevpc," doesn't exist in the hosted zone. Adding vpc..")
448484
hzid = update_hosted_zone(hostedzonename,regioname,instancevpc)
449-
make_ddb_entry(cluname,hzid,recordname,regioname) #Make ddb entry. This should only work from the calling region.
485+
make_ddb_entry(cluname,hzid,recordname,regioname,gdbobj) #Make ddb entry. This should only work from the calling region.
450486
else:
451487
print ("Vpc",instancevpc," doesn't exist. But skipping due to skip vpc flag.")
452488

453489
else:
454490
print ("Hosted Zone doesn't exists. Creating ",hostedzonename)
455491
hzid = create_hosted_zone(hostedzonename,regioname,instancevpc)
456-
make_ddb_entry(cluname,hzid,recordname,regioname) #Make ddb entry. This should only work from the calling region.
492+
make_ddb_entry(cluname,hzid,recordname,regioname,gdbobj) #Make ddb entry. This should only work from the calling region.
457493
else:
458494
print ("No entries made for this cluster. Cluster is not in the current region.")
459495

‎managed-gdb-cft.yml‎

Lines changed: 85 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ Description: Creates supporting resources for automated global database endpoint
66
# 8/29/2021 : ausamant - added managed policies for the lambda role instead of inline policies
77
# - added stackid as a uniquifier for managed policy names
88
# 4/23/2022 : ausamant - added ability to choose either planned or unplanned feature support
9+
# 5/14/2022 : ausamant - now removes clustername entries from all regions ddb table after unplanned failover
910
######################################################################################################################
1011

1112
## Parameters
@@ -77,6 +78,15 @@ Resources:
7778
import json
7879
import string
7980
import boto3
81+
from boto3.dynamodb.types import TypeSerializer,TypeDeserializer
82+
83+
def dsrl_ddb(dynamo_obj: dict) -> dict:
84+
deserializer = TypeDeserializer()
85+
return {
86+
k: deserializer.deserialize(v)
87+
for k, v in dynamo_obj.items()
88+
}
89+
8090
8191
def lambda_handler(event, context):
8292
@@ -165,15 +175,39 @@ Resources:
165175
print("This is a writer endpoint of a secondary region, skipping")
166176
167177
168-
# If this was a detach-promote event, we consider this as a unplanned failover and delete the ddb entry.
178+
# If this was a detach-promote event, we consider this as a unplanned failover and delete the ddb entry from all regions.
179+
# This makes sure no more subsequent gdb events are processed in any regions for this global cluster.
180+
169181
if eventid == "RDS-EVENT-0228":
170-
print("Removing entry for cluster",cluname,"from the dynamodb table")
171-
dresponse = ddbclient.delete_item(
172-
TableName='gdbcnamepair',
173-
Key = {
174-
'clustername':{'S':cluname}
175-
}
176-
)
182+
183+
dresponse = ddbclient.get_item(
184+
TableName='gdbcnamepair',
185+
Key= {"clustername": {"S": cluname}},
186+
ProjectionExpression ='allregions')
187+
188+
# deserialize map object to dict before processing
189+
gdbobj = dsrl_ddb(dresponse['Item'])
190+
gdbobj = (gdbobj['allregions'])
191+
192+
for gdbclu in (gdbobj):
193+
194+
# gdbobj has the data in format {'clustername':'regionname'}.
195+
# Extract cluster name and region name.
196+
gdbregion = gdbobj[gdbclu]
197+
cluname = gdbclu
198+
199+
#connect to each region by building the boto client in the loop
200+
gdbclientr = boto3.client("dynamodb",gdbregion)
201+
202+
print("Removing entry for cluster",cluname,"from the dynamodb table in region",gdbregion)
203+
204+
# delete the ddb entry from this region.
205+
# loop through all regions where the gdb cluster existsed.
206+
dresponse = gdbclientr.delete_item(
207+
TableName='gdbcnamepair',
208+
Key = {
209+
'clustername':{'S':cluname}
210+
})
177211

178212
return {
179213
'statusCode': 200,
@@ -332,6 +366,16 @@ Resources:
332366
import json
333367
import string
334368
import boto3
369+
from boto3.dynamodb.types import TypeSerializer,TypeDeserializer
370+
371+
def dsrl_ddb(dynamo_obj: dict) -> dict:
372+
deserializer = TypeDeserializer()
373+
return {
374+
k: deserializer.deserialize(v)
375+
for k, v in dynamo_obj.items()
376+
}
377+
378+
335379
336380
def lambda_handler(event, context):
337381
@@ -419,15 +463,38 @@ Resources:
419463
elif (j['EndpointType']=="WRITER" and j['Status']=='inactive'):
420464
print("This is a writer enpoint of a secondary region, skipping")
421465
422-
# Delete the ddb entry.
423-
print("Removing entry for cluster",cluname,"from the dynamodb table")
424-
dresponse = ddbclient.delete_item(
425-
TableName='gdbcnamepair',
426-
Key = {
427-
'clustername':{'S':cluname}
428-
}
429-
)
430-
466+
# If this was a detach-promote event, we consider this as a unplanned failover and delete the ddb entry from all regions.
467+
# This makes sure no more subsequent gdb events are processed in any regions for this global cluster.
468+
469+
dresponse = ddbclient.get_item(
470+
TableName='gdbcnamepair',
471+
Key= {"clustername": {"S": cluname}},
472+
ProjectionExpression ='allregions')
473+
474+
# deserialize map object to dict before processing
475+
gdbobj = dsrl_ddb(dresponse['Item'])
476+
gdbobj = (gdbobj['allregions'])
477+
478+
for gdbclu in (gdbobj):
479+
480+
# gdbobj has the data in format {'clustername':'regionname'}.
481+
# Extract cluster name and region name.
482+
gdbregion = gdbobj[gdbclu]
483+
cluname = gdbclu
484+
485+
#connect to each region by building the boto client in the loop
486+
gdbclientr = boto3.client("dynamodb",gdbregion)
487+
488+
print("Removing entry for cluster",cluname,"from the dynamodb table in region",gdbregion)
489+
490+
# delete the ddb entry from this region.
491+
# loop through all regions where the gdb cluster existed.
492+
dresponse = gdbclientr.delete_item(
493+
TableName='gdbcnamepair',
494+
Key = {
495+
'clustername':{'S':cluname}
496+
})
497+
431498
return {
432499
'statusCode': 200,
433500
'body': json.dumps('event processed')
@@ -644,7 +711,7 @@ Resources:
644711
- dynamodb:PutItem
645712
- dynamodb:DeleteItem
646713
- dynamodb:UpdateItem
647-
Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${gdbmanagedepddbtbl}"
714+
Resource: !Sub "arn:aws:dynamodb:*:${AWS::AccountId}:table/${gdbmanagedepddbtbl}"
648715
ManagedPolicyName:
649716
Fn::Join:
650717
- '-'

0 commit comments

Comments
 (0)