Merge branch 'db-data-type' into 'enterprise'

feat(data-type): track database data type for columns

See merge request dkinternal/testgen/dataops-testgen!327

Author: ci bot

testgen/commands/queries/profiling_query.py

@@ -22,6 +22,7 @@ class CProfilingSQL:
     col_name = ""
     col_gen_type = ""
     col_type = ""
+    db_data_type = ""
     col_ordinal_position = "0"
     col_is_decimal = ""
     col_top_freq_update = ""
@@ -99,6 +100,7 @@ def _get_params(self) -> dict:
             "COL_NAME_SANITIZED": self.col_name.replace("'", "''"),
             "COL_GEN_TYPE": self.col_gen_type,
             "COL_TYPE": self.col_type or "",
+            "DB_DATA_TYPE": self.db_data_type or "",
             "COL_POS": self.col_ordinal_position,
             "TOP_FREQ": self.col_top_freq_update,
             "PROFILE_RUN_ID": self.profile_run_id,

testgen/commands/run_profiling_bridge.py

@@ -317,7 +317,8 @@ def run_profiling_queries(table_group_id: str, username: str | None = None, spin
                 clsProfiling.data_schema = dctColumnRecord["table_schema"]
                 clsProfiling.data_table = dctColumnRecord["table_name"]
                 clsProfiling.col_name = dctColumnRecord["column_name"]
-                clsProfiling.col_type = dctColumnRecord["data_type"]
+                clsProfiling.col_type = dctColumnRecord["column_type"]
+                clsProfiling.db_data_type = dctColumnRecord["db_data_type"]
                 clsProfiling.profile_run_id = profiling_run_id
                 clsProfiling.col_is_decimal = dctColumnRecord["is_decimal"]
                 clsProfiling.col_ordinal_position = dctColumnRecord["ordinal_position"]

testgen/commands/run_refresh_data_chars.py

@@ -50,6 +50,7 @@ def run_refresh_data_chars_queries(params: TestExecutionParams, run_date: str, s
         "position",
         "general_type",
         "column_type",
+        "db_data_type",
         "record_ct",
     ]
     staging_records = [
@@ -62,7 +63,8 @@ def run_refresh_data_chars_queries(params: TestExecutionParams, run_date: str, s
             item["column_name"],
             item["ordinal_position"],
             item["general_type"],
-            item["data_type"],
+            item["column_type"],
+            item["db_data_type"],
             count_map.get(f"{item['table_schema']}.{item['table_name']}", 0),
         ]
         for item in ddf_results

testgen/template/data_chars/data_chars_update.sql

@@ -120,6 +120,7 @@ WITH new_chars AS (
       position,
       general_type,
       column_type,
+      db_data_type,
       functional_data_type,
       run_date
    FROM {SOURCE_TABLE}
@@ -130,8 +131,9 @@ update_chars AS (
    SET ordinal_position = n.position,
       general_type = n.general_type,
       column_type = n.column_type,
+      db_data_type = n.db_data_type,
       functional_data_type = COALESCE(n.functional_data_type, d.functional_data_type),
-      last_mod_date = CASE WHEN n.column_type <> d.column_type THEN n.run_date ELSE d.last_mod_date END,
+      last_mod_date = CASE WHEN n.db_data_type <> d.db_data_type THEN n.run_date ELSE d.last_mod_date END,
       drop_date = NULL
    FROM new_chars n
       INNER JOIN data_column_chars d ON (
@@ -142,22 +144,22 @@ update_chars AS (
       )
    WHERE data_column_chars.table_id = d.table_id
       AND data_column_chars.column_name = d.column_name
-   RETURNING data_column_chars.*, d.column_type as old_column_type
+   RETURNING data_column_chars.*, d.db_data_type as old_data_type
 )
 INSERT INTO data_structure_log (
    element_id,
    change_date,
    change,
-   old_column_type,
-   new_column_type
+   old_data_type,
+   new_data_type
 )
 SELECT u.column_id,
    u.last_mod_date,
    'M',
-   u.old_column_type,
-   u.column_type
+   u.old_data_type,
+   u.db_data_type
    FROM update_chars u
-   WHERE u.old_column_type <> u.column_type;
+   WHERE u.old_data_type <> u.db_data_type;
 
 
 -- Add new records
@@ -169,6 +171,7 @@ WITH new_chars AS (
       position,
       general_type,
       column_type,
+      db_data_type,
       functional_data_type,
       run_date
    FROM {SOURCE_TABLE}
@@ -184,6 +187,7 @@ inserted_records AS (
          ordinal_position,
          general_type,
          column_type,
+         db_data_type,
          functional_data_type,
          add_date,
          last_mod_date
@@ -196,6 +200,7 @@ inserted_records AS (
       n.position,
       n.general_type,
       n.column_type,
+      n.db_data_type,
       n.functional_data_type,
       n.run_date,
       n.run_date
@@ -218,12 +223,12 @@ INSERT INTO data_structure_log (
    element_id,
    change_date,
    change,
-   new_column_type
+   new_data_type
 )
 SELECT i.column_id,
    i.add_date,
    'A',
-   i.column_type
+   i.db_data_type
    FROM inserted_records i;
 
 -- Mark dropped records
@@ -263,10 +268,10 @@ INSERT INTO data_structure_log (
    element_id,
    change_date,
    change,
-   old_column_type
+   old_data_type
 )
 SELECT del.column_id,
    del.drop_date,
    'D',
-   del.column_type
+   del.db_data_type
    FROM deleted_records del;

testgen/template/dbsetup/030_initialize_new_schema_structure.sql

@@ -40,6 +40,7 @@ CREATE TABLE stg_data_chars_updates (
    position              INTEGER,
    general_type          VARCHAR(1),
    column_type           VARCHAR(50),
+   db_data_type          VARCHAR(50),
    functional_data_type  VARCHAR(50),
    record_ct             BIGINT
 );
@@ -247,6 +248,7 @@ CREATE TABLE profile_results (
    position              INTEGER,
    column_name           VARCHAR(120),
    column_type           VARCHAR(50),
+   db_data_type          VARCHAR(50),
    general_type          VARCHAR(1),
    record_ct             BIGINT,
    value_ct              BIGINT,
@@ -340,6 +342,7 @@ CREATE TABLE profile_anomaly_results (
    table_name      VARCHAR(120),
    column_name     VARCHAR(500),
    column_type     VARCHAR(50),
+   db_data_type    VARCHAR(50),
    anomaly_id      VARCHAR(10),
    detail          VARCHAR,
    disposition     VARCHAR(20), -- Confirmed, Dismissed, Inactive
@@ -369,11 +372,11 @@ CREATE TABLE data_structure_log (
    log_id UUID DEFAULT gen_random_uuid()
       CONSTRAINT pk_dsl_id
          PRIMARY KEY,
-   element_id UUID,
-   change_date TIMESTAMP,
-   change VARCHAR(10),
-   old_column_type VARCHAR(50),
-   new_column_type VARCHAR(50)
+   element_id    UUID,
+   change_date   TIMESTAMP,
+   change        VARCHAR(10),
+   old_data_type VARCHAR(50),
+   new_data_type VARCHAR(50)
 );
 
 CREATE TABLE data_table_chars (
@@ -418,6 +421,7 @@ CREATE TABLE data_column_chars (
    ordinal_position       INTEGER,
    general_type           VARCHAR(1),
    column_type            VARCHAR(50),
+   db_data_type           VARCHAR(50),
    functional_data_type   VARCHAR(50),
    description            VARCHAR(1000),
    critical_data_element  BOOLEAN,

testgen/template/dbupgrade/0156_incremental_upgrade.sql

@@ -0,0 +1,21 @@
+SET SEARCH_PATH TO {SCHEMA_NAME};
+
+ALTER TABLE data_structure_log RENAME COLUMN old_column_type TO old_data_type;
+ALTER TABLE data_structure_log RENAME COLUMN new_column_type TO new_data_type;
+
+ALTER TABLE stg_data_chars_updates ADD COLUMN db_data_type VARCHAR(50);
+ALTER TABLE profile_results ADD COLUMN db_data_type VARCHAR(50);
+ALTER TABLE profile_anomaly_results ADD COLUMN db_data_type VARCHAR(50);
+ALTER TABLE data_column_chars ADD COLUMN db_data_type VARCHAR(50);
+
+UPDATE profile_results
+    SET db_data_type = column_type
+    WHERE db_data_type IS NULL;
+
+UPDATE profile_anomaly_results
+    SET db_data_type = column_type
+    WHERE db_data_type IS NULL;
+
+UPDATE data_column_chars
+    SET db_data_type = column_type
+    WHERE db_data_type IS NULL;

testgen/template/flavors/bigquery/data_chars/schema_ddf_query_bigquery.sql

@@ -8,7 +8,8 @@ SELECT '{PROJECT_CODE}' AS project_code,
            WHEN LOWER(c.data_type) = 'date' THEN 'date'
            WHEN LOWER(c.data_type) = 'bool' THEN 'boolean'
            ELSE LOWER(c.data_type)
-       END AS data_type,
+       END AS column_type,
+       c.data_type AS db_data_type,
        NULL AS character_maximum_length,
        c.ordinal_position,
        CASE

testgen/template/flavors/bigquery/profiling/project_profiling_query_bigquery.yaml

@@ -19,6 +19,7 @@ strTemplate01_5: |
   {COL_POS} AS position,
   '{COL_NAME_SANITIZED}' AS column_name,
   '{COL_TYPE}' AS column_type,
+  '{DB_DATA_TYPE}' AS db_data_type,
   '{COL_GEN_TYPE}' AS general_type,
   COUNT(*) AS record_ct,
 strTemplate02_X: |

testgen/template/flavors/databricks/data_chars/schema_ddf_query_databricks.sql

@@ -9,7 +9,8 @@ SELECT '{PROJECT_CODE}' AS project_code,
            WHEN lower(c.full_data_type) IN ('double', 'float') THEN 'numeric'
            WHEN lower(c.full_data_type) LIKE 'decimal%' THEN 'numeric(' || c.numeric_precision || ',' || c.numeric_scale || ')'
            ELSE lower(c.full_data_type)
-       END AS data_type,
+       END AS column_type,
+       c.full_data_type AS db_data_type,
        c.character_maximum_length,
        c.ordinal_position,
        CASE

testgen/template/flavors/databricks/profiling/project_profiling_query_databricks.yaml

@@ -11,6 +11,7 @@ strTemplate01_5: |
   {COL_POS} AS position,
   '{COL_NAME_SANITIZED}' AS column_name,
   '{COL_TYPE}' AS column_type,
+  '{DB_DATA_TYPE}' AS db_data_type,
   '{COL_GEN_TYPE}' AS general_type,
   COUNT(*) AS record_ct,
 strTemplate02_X: |