|
| 1 | +import logging |
| 2 | +import time |
| 3 | + |
| 4 | +import boto3 |
| 5 | +from typing import Dict |
| 6 | + |
| 7 | + |
| 8 | +class SSMManager: |
| 9 | + logger = logging.getLogger('sagemaker-ssh-helper:SSMManager') |
| 10 | + |
| 11 | + def __init__(self, region_name=None, sleep_between_retries_in_seconds=10, redo_attempts=5, |
| 12 | + clock_timestamp_override=None) -> None: |
| 13 | + super().__init__() |
| 14 | + self.clock_timestamp_override = clock_timestamp_override |
| 15 | + self.redo_attempts = redo_attempts |
| 16 | + self.sleep_between_retries_in_seconds = sleep_between_retries_in_seconds |
| 17 | + self.region_name = region_name |
| 18 | + |
| 19 | + def list_all_instances_with_tags(self) -> Dict[str, Dict[str, str]]: |
| 20 | + ssm = boto3.client('ssm', region_name=self.region_name) |
| 21 | + |
| 22 | + result = {} |
| 23 | + next_token = "" # nosec hardcoded_password_string # not a password |
| 24 | + while next_token is not None: |
| 25 | + response = ssm.describe_instance_information( |
| 26 | + Filters=[{'Key': 'ResourceType', 'Values': ['ManagedInstance']}], |
| 27 | + NextToken=next_token, |
| 28 | + MaxResults=50, |
| 29 | + ) |
| 30 | + next_token = response.get('NextToken') |
| 31 | + info_list = response['InstanceInformationList'] |
| 32 | + if info_list: |
| 33 | + for info in info_list: |
| 34 | + instance_id = info['InstanceId'] |
| 35 | + tags = ssm.list_tags_for_resource(ResourceType='ManagedInstance', ResourceId=instance_id) |
| 36 | + tags_dict = {} |
| 37 | + if 'TagList' in tags: |
| 38 | + for tag in tags['TagList']: |
| 39 | + tags_dict[tag['Key']] = tag['Value'] |
| 40 | + tags_dict['$__SSMManager__.PingStatus'] = info['PingStatus'] |
| 41 | + result[instance_id] = tags_dict |
| 42 | + |
| 43 | + return result |
| 44 | + |
| 45 | + def get_training_instance_ids(self, training_job_name, timeout_in_sec=0, expected_count=1): |
| 46 | + self.logger.info(f"Querying SSM instance IDs for training job {training_job_name}, " |
| 47 | + f"expected instance count = {expected_count}") |
| 48 | + return self.get_instance_ids('training-job', training_job_name, timeout_in_sec, |
| 49 | + expected_count) |
| 50 | + |
| 51 | + def get_processing_instance_ids(self, processing_job_name, timeout_in_sec=0): |
| 52 | + self.logger.info(f"Querying SSM instance IDs for processing job {processing_job_name}") |
| 53 | + return self.get_instance_ids('processing-job', processing_job_name, timeout_in_sec) |
| 54 | + |
| 55 | + def get_endpoint_instance_ids(self, endpoint_name, timeout_in_sec=0): |
| 56 | + raise AssertionError("Not supported yet.") |
| 57 | + |
| 58 | + def get_transformer_instance_ids(self, transform_job_name, timeout_in_sec=0): |
| 59 | + self.logger.info(f"Querying SSM instance IDs for transform job {transform_job_name}") |
| 60 | + return self.get_instance_ids('transform-job', transform_job_name, timeout_in_sec) |
| 61 | + |
| 62 | + def get_studio_kgw_instance_ids(self, kgw_name, timeout_in_sec=0): |
| 63 | + self.logger.info(f"Querying SSM instance IDs for SageMaker Studio kernel gateway {kgw_name}") |
| 64 | + return self.get_instance_ids('app', f"{kgw_name}", timeout_in_sec) |
| 65 | + |
| 66 | + def get_notebook_instance_ids(self, instance_name, timeout_in_sec=0): |
| 67 | + self.logger.info(f"Querying SSM instance IDs for SageMaker notebook instance {instance_name}") |
| 68 | + return self.get_instance_ids('notebook-instance', f"{instance_name}", timeout_in_sec) |
| 69 | + |
| 70 | + def get_instance_ids_once(self, arn_resource_type, arn_resource_name): |
| 71 | + all_instances = self.list_all_instances_with_tags() |
| 72 | + result_pairs = [] |
| 73 | + for mi_id in all_instances: |
| 74 | + tags = all_instances[mi_id] |
| 75 | + if "SSHResourceName" not in tags or "SSHResourceArn" not in tags: |
| 76 | + continue |
| 77 | + if f"/{arn_resource_name}" in tags["SSHResourceArn"] and \ |
| 78 | + arn_resource_name == tags["SSHResourceName"] and \ |
| 79 | + f":{arn_resource_type}/" in tags["SSHResourceArn"]: |
| 80 | + if "SSHTimestamp" in tags: |
| 81 | + timestamp = tags["SSHTimestamp"] |
| 82 | + else: |
| 83 | + timestamp = 0 |
| 84 | + result_pairs.append((mi_id, timestamp)) |
| 85 | + |
| 86 | + result_pairs.sort(key=lambda i: i[1], reverse=True) |
| 87 | + result = [i[0] for i in result_pairs] |
| 88 | + return result |
| 89 | + |
| 90 | + def get_instance_ids(self, arn_resource_type, arn_resource_name, |
| 91 | + timeout_in_sec=0, |
| 92 | + expected_count=1): |
| 93 | + mi_ids = self.get_instance_ids_once(arn_resource_type, arn_resource_name) |
| 94 | + |
| 95 | + while not mi_ids and timeout_in_sec > 0: |
| 96 | + self.logger.info(f"SSH Helper not yet started? Retrying. Seconds left: {timeout_in_sec}") |
| 97 | + time.sleep(self.sleep_between_retries_in_seconds) |
| 98 | + mi_ids = self.get_instance_ids_once(arn_resource_type, arn_resource_name) |
| 99 | + timeout_in_sec -= self.sleep_between_retries_in_seconds |
| 100 | + |
| 101 | + self.logger.info(f"Got preliminary SSM instance IDs: {mi_ids}") |
| 102 | + |
| 103 | + redo_attempts = self.redo_attempts |
| 104 | + # noinspection DuplicatedCode |
| 105 | + while len(mi_ids) < expected_count and redo_attempts > 0: |
| 106 | + self.logger.info(f"Re-fetch results for other instances to catchup. Attempts left: {redo_attempts}") |
| 107 | + time.sleep(30) |
| 108 | + mi_ids = self.get_instance_ids_once(arn_resource_type, arn_resource_name) |
| 109 | + redo_attempts -= 1 |
| 110 | + |
| 111 | + self.logger.info(f"Got final SSM instance IDs: {mi_ids}") |
| 112 | + return mi_ids |
| 113 | + |
| 114 | + def list_expired_ssh_instances(self, expiration_days=0): |
| 115 | + all_instances = self.list_all_instances_with_tags() |
| 116 | + logging.info("Found %s instances in SSM", len(all_instances)) |
| 117 | + |
| 118 | + expired_instances = [] |
| 119 | + for mi_id in all_instances: |
| 120 | + tags = all_instances[mi_id] |
| 121 | + if "SSHTimestamp" in tags: |
| 122 | + timestamp = int(tags["SSHTimestamp"]) |
| 123 | + else: |
| 124 | + timestamp = 0 |
| 125 | + if "$__SSMManager__.PingStatus" in tags: |
| 126 | + ping_status = tags["$__SSMManager__.PingStatus"] |
| 127 | + else: |
| 128 | + ping_status = "Online" |
| 129 | + if ping_status == "Online": |
| 130 | + continue |
| 131 | + if self.clock_timestamp_override is not None: |
| 132 | + expiration_timestamp = self.clock_timestamp_override |
| 133 | + else: |
| 134 | + expiration_timestamp = int(round(time.time())) |
| 135 | + expiration_timestamp -= expiration_days * 3600 * 24 |
| 136 | + if timestamp < expiration_timestamp: |
| 137 | + expired_instances.append(mi_id) |
| 138 | + logging.info("Found expired offline SSH instance %s with timestamp %s", mi_id, timestamp) |
| 139 | + |
| 140 | + logging.info("Found %s expired offline SSH instances", len(expired_instances)) |
| 141 | + return expired_instances |
0 commit comments