As Glue limits comments to 255 characters, we may need to truncate them.

Author: mikko-leppanen-fonecta-com

utilities/Hive_metastore_migration/src/hive_metastore_migration.py

@@ -414,6 +414,14 @@ def transform_ms_bucketing_cols(self, ms_bucketing_cols):
                                           payload_func=lambda row: row['BUCKET_COL_NAME'])
 
     def transform_ms_columns(self, ms_columns):
+        def extract_row(row):
+            def truncate(x):
+                return x[:255] if hasattr(x,"__getitem__") else x
+            return (
+                row['COLUMN_NAME'],
+                row['TYPE_NAME'],
+                truncate(row['COMMENT'])
+            )
         return self.transform_df_with_idx(df=ms_columns,
                                           id_col='CD_ID',
                                           idx='INTEGER_IDX',
@@ -422,8 +430,7 @@ def transform_ms_columns(self, ms_columns):
                                               StructField(name='name', dataType=StringType()),
                                               StructField(name='type', dataType=StringType()),
                                               StructField(name='comment', dataType=StringType())]),
-                                          payload_func=lambda row: (
-                                              row['COLUMN_NAME'], row['TYPE_NAME'], row['COMMENT']))
+                                          payload_func=extract_row)
 
     def transform_ms_skewed_col_names(self, ms_skewed_col_names):
         return self.transform_df_with_idx(df=ms_skewed_col_names,