Skip to content

Commit 5fdeeda

Browse files
author
ci bot
committed
Merge branch 'aarthy/test-validation' into 'enterprise'
fix(test-validation): detect missing columns in Dupe_Rows test See merge request dkinternal/testgen/dataops-testgen!328
2 parents 1f6ad69 + a1500cd commit 5fdeeda

File tree

7 files changed

+18
-11
lines changed

7 files changed

+18
-11
lines changed

‎testgen/commands/queries/test_parameter_validation_query.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ def _get_query(self, template_file_name: str, sub_directory: str | None = "valid
3535
"CAT_TEST_IDS": tuple(self.test_ids or []),
3636
"START_TIME": self.today,
3737
"NOW_TIMESTAMP": date_service.get_now_as_string(),
38-
"COLUMNS_TABLE": f"{self.tg_schema}.INFORMATION_SCHEMA.COLUMNS" if self.flavor == "bigquery" else "information_schema.columns"
38+
"COLUMNS_TABLE": f"{self.tg_schema}.INFORMATION_SCHEMA.COLUMNS" if self.flavor == "bigquery" else "information_schema.columns",
39+
"ID_SEPARATOR": "`" if self.flavor in ("databricks", "bigquery") else '"',
3940
}
4041
query = replace_params(query, params)
4142
return query, params

‎testgen/commands/run_launch_db_config.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ def run_launch_db_config(delete_db: bool, drop_users_and_roles: bool = True) ->
8686
user_override=params_mapping["TESTGEN_ADMIN_USER"],
8787
password_override=params_mapping["TESTGEN_ADMIN_PASSWORD"],
8888
user_type="schema_admin",
89+
suppress_logs=True,
8990
)
9091
import_metadata_records_from_yaml(params_mapping)
9192

‎testgen/commands/run_upgrade_db_config.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ def _refresh_static_metadata(params_mapping):
9696
user_override=params_mapping["TESTGEN_ADMIN_USER"],
9797
password_override=params_mapping["TESTGEN_ADMIN_PASSWORD"],
9898
user_type="schema_admin",
99+
suppress_logs=True,
99100
)
100101
import_metadata_records_from_yaml(params_mapping)
101102

@@ -106,6 +107,7 @@ def _refresh_static_metadata(params_mapping):
106107
user_override=params_mapping["TESTGEN_ADMIN_USER"],
107108
password_override=params_mapping["TESTGEN_ADMIN_PASSWORD"],
108109
user_type="schema_admin",
110+
suppress_logs=True,
109111
)
110112

111113

‎testgen/common/database/database_service.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -134,8 +134,9 @@ def execute_db_queries(
134134
user_override: str | None = None,
135135
password_override: str | None = None,
136136
user_type: UserType = "normal",
137+
suppress_logs: bool = False,
137138
) -> tuple[list[Any], list[int]]:
138-
LOG.info(f"DB operation: execute_db_queries on {'Target' if use_target_db else 'App'} database (User type = {user_type})")
139+
LOG.info(f"DB operation: execute_db_queries ({len(queries)}) on {'Target' if use_target_db else 'App'} database (User type = {user_type})")
139140

140141
with _init_db_connection(use_target_db, user_override, password_override, user_type) as connection:
141142
return_values: list[Any] = []
@@ -144,7 +145,8 @@ def execute_db_queries(
144145
LOG.info("No queries to process")
145146
for index, (query, params) in enumerate(queries):
146147
LOG.debug(f"Query: {query}")
147-
LOG.info(f"Processing {index + 1} of {len(queries)} queries")
148+
if not suppress_logs:
149+
LOG.info(f"Processing {index + 1} of {len(queries)} queries")
148150
transaction = connection.begin()
149151
result = connection.execute(text(query), params)
150152
row_counts.append(result.rowcount)

‎testgen/common/read_yaml_metadata_records.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ def _process_yaml_for_import(params_mapping: dict, data:dict, parent_table:str,
164164
user_override=params_mapping["TESTGEN_ADMIN_USER"],
165165
password_override=params_mapping["TESTGEN_ADMIN_PASSWORD"],
166166
user_type="schema_admin",
167+
suppress_logs=True,
167168
)
168169
return
169170

‎testgen/template/dbsetup_test_types/test_types_Incr_Avg_Shift.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ test_types:
5656
test_type: Incr_Avg_Shift
5757
sql_flavor: mssql
5858
measure: |-
59-
COALESCE(ABS( ({BASELINE_AVG} - (SUM({COLUMN_NAME}) - {BASELINE_SUM}) / NULLIF(CAST(COUNT({COLUMN_NAME}) AS FLOAT) - {BASELINE_VALUE_CT}, 0)) / {BASELINE_SD} ), 0)
59+
COALESCE(ABS( ({BASELINE_AVG} - (SUM(CAST("{COLUMN_NAME}" AS FLOAT)) - {BASELINE_SUM}) / NULLIF(CAST(COUNT({COLUMN_NAME}) AS FLOAT) - {BASELINE_VALUE_CT}, 0)) / {BASELINE_SD} ), 0)
6060
test_operator: '>='
6161
test_condition: |-
6262
{THRESHOLD_VALUE}

‎testgen/template/validate_tests/ex_get_test_column_list_tg.sql‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
SELECT cat_test_id,
1818
schema_name AS schema_name,
1919
table_name AS table_name,
20-
TRIM(UNNEST(STRING_TO_ARRAY(column_name, ','))) as column_name
20+
TRIM(TRIM(UNNEST(STRING_TO_ARRAY(column_name, ','))), '{ID_SEPARATOR}') as column_name
2121
FROM test_definitions d
2222
INNER JOIN test_types t
2323
ON d.test_type = t.test_type
@@ -26,23 +26,23 @@
2626
AND t.test_scope = 'referential'
2727
AND t.test_type NOT LIKE 'Aggregate_%'
2828
UNION
29-
-- FROM: groupby_names (should be referential)
29+
-- FROM: groupby_names
3030
SELECT cat_test_id,
3131
schema_name AS schema_name,
3232
table_name AS table_name,
33-
TRIM(UNNEST(STRING_TO_ARRAY(groupby_names, ','))) as column_name
33+
TRIM(TRIM(UNNEST(STRING_TO_ARRAY(groupby_names, ','))), '{ID_SEPARATOR}') as column_name
3434
FROM test_definitions d
3535
INNER JOIN test_types t
3636
ON d.test_type = t.test_type
3737
WHERE test_suite_id = :TEST_SUITE_ID
3838
AND COALESCE(test_active, 'Y') = 'Y'
39-
AND t.test_scope IN ('column', 'referential')
39+
AND t.test_scope IN ('column', 'referential', 'table')
4040
UNION
4141
-- FROM: window_date_column (referential)
4242
SELECT cat_test_id,
4343
schema_name AS schema_name,
4444
table_name AS table_name,
45-
TRIM(UNNEST(STRING_TO_ARRAY(window_date_column, ','))) as column_name
45+
TRIM(TRIM(UNNEST(STRING_TO_ARRAY(window_date_column, ','))), '{ID_SEPARATOR}') as column_name
4646
FROM test_definitions d
4747
INNER JOIN test_types t
4848
ON d.test_type = t.test_type
@@ -54,7 +54,7 @@
5454
SELECT cat_test_id,
5555
match_schema_name AS schema_name,
5656
match_table_name AS table_name,
57-
TRIM(UNNEST(STRING_TO_ARRAY(match_column_names, ','))) as column_name
57+
TRIM(TRIM(UNNEST(STRING_TO_ARRAY(match_column_names, ','))), '{ID_SEPARATOR}') as column_name
5858
FROM test_definitions d
5959
INNER JOIN test_types t
6060
ON d.test_type = t.test_type
@@ -67,7 +67,7 @@
6767
SELECT cat_test_id,
6868
match_schema_name AS schema_name,
6969
match_table_name AS table_name,
70-
TRIM(UNNEST(STRING_TO_ARRAY(match_groupby_names, ','))) as column_name
70+
TRIM(TRIM(UNNEST(STRING_TO_ARRAY(match_groupby_names, ','))), '{ID_SEPARATOR}') as column_name
7171
FROM test_definitions d
7272
INNER JOIN test_types t
7373
ON d.test_type = t.test_type

0 commit comments

Comments
 (0)