From f748d06e13c293864741c4f96820446de31df041 Mon Sep 17 00:00:00 2001 From: pritishpai Date: Mon, 4 Aug 2025 11:12:29 -0400 Subject: [PATCH 01/13] Upgrade databricks sdk requirement to 0.62.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5cff256b28..35693cce9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ classifiers = [ ] -dependencies = ["databricks-sdk>=0.58.0,<0.59.0", +dependencies = ["databricks-sdk>=0.61.0,<0.62.0", "databricks-labs-lsql>=0.16.0,<0.17.0", "databricks-labs-blueprint>=0.11.0,<0.12.0", "PyYAML>=6.0.0,<6.1.0", From a892de95796b63fe2fceb2e0d7c7f8f4818fac0f Mon Sep 17 00:00:00 2001 From: pritishpai Date: Mon, 4 Aug 2025 11:25:22 -0400 Subject: [PATCH 02/13] Remove fixture as legacy dashboard creation is now deprecated. --- tests/integration/conftest.py | 62 +---------------------------------- 1 file changed, 1 insertion(+), 61 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1ba4478dd9..bbafb133dd 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -29,7 +29,7 @@ from databricks.sdk.service.dashboards import Dashboard as SdkLakeviewDashboard from databricks.sdk.service.iam import Group from databricks.sdk.service.jobs import Job, SparkPythonTask -from databricks.sdk.service.sql import Dashboard as SdkRedashDashboard, WidgetPosition, WidgetOptions, LegacyQuery +from databricks.sdk.service.sql import Dashboard as SdkRedashDashboard, LegacyQuery from databricks.labs.ucx.__about__ import __version__ from databricks.labs.ucx.account.workspaces import AccountWorkspaces @@ -135,66 +135,6 @@ def delete(dashboard: SdkLakeviewDashboard) -> None: yield from factory("dashboard", create, delete) -@pytest.fixture -def make_dashboard( - ws: WorkspaceClient, - make_random: Callable[[int], str], - make_query, - watchdog_purge_suffix, -): - """Create a legacy dashboard. - This fixture is used to test migrating legacy dashboards to Lakeview. - """ - - def create(query: LegacyQuery | None = None) -> SdkRedashDashboard: - if not query: - query = make_query() - assert query - assert query.id - viz = ws.query_visualizations_legacy.create( - type="table", - query_id=query.id, - options={ - "itemsPerPage": 1, - "condensed": True, - "withRowNumber": False, - "version": 2, - "columns": [ - {"name": "id", "title": "id", "allowSearch": True}, - ], - }, - ) - - dashboard_name = f"ucx_D{make_random(4)}_{watchdog_purge_suffix}" - dashboard = ws.dashboards.create(name=dashboard_name, tags=["original_dashboard_tag"]) - assert dashboard.id is not None - ws.dashboard_widgets.create( - dashboard_id=dashboard.id, - visualization_id=viz.id, - width=1, - options=WidgetOptions( - title="", - position=WidgetPosition( - col=0, - row=0, - size_x=3, - size_y=3, - ), - ), - ) - logger.info(f"Dashboard Created {dashboard_name}: {ws.config.host}/sql/dashboards/{dashboard.id}") - return ws.dashboards.get(dashboard.id) # Dashboard with widget - - def remove(dashboard: SdkRedashDashboard) -> None: - try: - assert dashboard.id is not None - ws.dashboards.delete(dashboard_id=dashboard.id) - except RuntimeError as e: - logger.info(f"Can't delete dashboard {e}") - - yield from factory("dashboard", create, remove) - - @pytest.fixture def make_dbfs_data_copy(ws, make_cluster, env_or_skip): _ = make_cluster # Need cluster to copy data From a89284df9d4ea8d432bf07795977aa334fb55601 Mon Sep 17 00:00:00 2001 From: pritishpai Date: Mon, 4 Aug 2025 14:05:50 -0400 Subject: [PATCH 03/13] Remove make_dashboard from the fixtures --- tests/integration/conftest.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index bbafb133dd..2c3e87869d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -407,7 +407,6 @@ def __init__( # pylint: disable=too-many-arguments make_job_fixture, make_notebook_fixture, make_query_fixture, - make_dashboard_fixture, make_lakeview_dashboard_fixture, make_cluster_policy_fixture, make_cluster_policy_permissions_fixture, @@ -432,7 +431,6 @@ def __init__( # pylint: disable=too-many-arguments self._make_job = make_job_fixture self._make_notebook = make_notebook_fixture self._make_query = make_query_fixture - self._make_dashboard = make_dashboard_fixture self._make_lakeview_dashboard = make_lakeview_dashboard_fixture self._make_cluster_policy = make_cluster_policy_fixture self._make_cluster_policy_permissions = make_cluster_policy_permissions_fixture @@ -958,7 +956,6 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals make_job_fixture, make_notebook_fixture, make_query_fixture, - make_dashboard_fixture, make_lakeview_dashboard_fixture, make_cluster_policy, make_cluster_policy_permissions, @@ -976,7 +973,6 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals make_job_fixture, make_notebook_fixture, make_query_fixture, - make_dashboard_fixture, make_lakeview_dashboard_fixture, make_cluster_policy, make_cluster_policy_permissions, @@ -1158,7 +1154,6 @@ def installation_ctx( # pylint: disable=too-many-arguments,too-many-locals make_job, make_notebook, make_query, - make_dashboard, make_lakeview_dashboard, make_cluster_policy, make_cluster_policy_permissions, @@ -1179,7 +1174,6 @@ def installation_ctx( # pylint: disable=too-many-arguments,too-many-locals make_job, make_notebook, make_query, - make_dashboard, make_lakeview_dashboard, make_cluster_policy, make_cluster_policy_permissions, From f364b016316b02da172f0e967f2bfe234a9400be Mon Sep 17 00:00:00 2001 From: pritishpai Date: Mon, 4 Aug 2025 14:06:42 -0400 Subject: [PATCH 04/13] Modify existing test test_query_linter_lints_queries_and_stores_dfsas_and_tables to not use make_dashboard --- tests/integration/source_code/test_queries.py | 48 +++---------------- 1 file changed, 7 insertions(+), 41 deletions(-) diff --git a/tests/integration/source_code/test_queries.py b/tests/integration/source_code/test_queries.py index e993d49e5d..250e17cb9c 100644 --- a/tests/integration/source_code/test_queries.py +++ b/tests/integration/source_code/test_queries.py @@ -3,22 +3,18 @@ from databricks.labs.ucx.source_code.base import DirectFsAccess, LineageAtom, UsedTable - -@pytest.mark.skip(reason="Legacy dashboard creation is no longer supported by Databricks.") def test_query_linter_lints_queries_and_stores_dfsas_and_tables(simple_ctx) -> None: query_with_dfsa = simple_ctx.make_query(sql_query="SELECT * from csv.`dbfs://some_folder/some_file.csv`") - dashboard_with_dfsa = simple_ctx.make_dashboard(query=query_with_dfsa) # Lakeview dashboard expects a string, not a legacy query - dashboard_with_used_table = simple_ctx.make_lakeview_dashboard(query="SELECT * FROM some_schema.some_table") simple_ctx.query_linter.refresh_report() problems = list(simple_ctx.sql_backend.fetch("SELECT * FROM query_problems", schema=simple_ctx.inventory_database)) assert problems == [ Row( - dashboard_id=dashboard_with_dfsa.id, - dashboard_parent=dashboard_with_dfsa.parent, - dashboard_name=dashboard_with_dfsa.name, + dashboard_id=None, + dashboard_parent=None, + dashboard_name=None, query_id=query_with_dfsa.id, query_parent=query_with_dfsa.parent, query_name=query_with_dfsa.name, @@ -31,19 +27,13 @@ def test_query_linter_lints_queries_and_stores_dfsas_and_tables(simple_ctx) -> N # By comparing the element instead of the list the `field(compare=False)` of the dataclass attributes take effect assert dfsas == [ DirectFsAccess( - source_id=f"{dashboard_with_dfsa.id}/{query_with_dfsa.id}", + source_id=f"no-dashboard-id/{query_with_dfsa.id}", source_lineage=[ - LineageAtom( - object_type="DASHBOARD", - object_id=dashboard_with_dfsa.id, - other={"parent": dashboard_with_dfsa.parent, "name": dashboard_with_dfsa.name}, - ), LineageAtom( object_type="QUERY", - object_id=f"{dashboard_with_dfsa.id}/{query_with_dfsa.id}", + object_id=f"no-dashboard-id/{query_with_dfsa.id}", other={"name": query_with_dfsa.name}, - ), - ], + )], path="dbfs://some_folder/some_file.csv", is_read=True, is_write=False, @@ -54,28 +44,4 @@ def test_query_linter_lints_queries_and_stores_dfsas_and_tables(simple_ctx) -> N # By comparing the element instead of the list the `field(compare=False)` of the dataclass attributes take effect # The "query" in the source and object id, and "count" in the name are hardcoded in the # `make_lakeview_dashboard` fixture - assert used_tables == [ - UsedTable( - source_id=f"{dashboard_with_used_table.dashboard_id}/query", - source_lineage=[ - LineageAtom( - object_type="DASHBOARD", - object_id=dashboard_with_used_table.dashboard_id, - other={ - "parent": dashboard_with_used_table.parent_path, - "name": dashboard_with_used_table.display_name, - }, - ), - LineageAtom( - object_type="QUERY", - object_id=f"{dashboard_with_used_table.dashboard_id}/query", - other={"name": "count"}, - ), - ], - catalog_name="hive_metastore", - schema_name="some_schema", - table_name="some_table", - is_read=True, - is_write=False, - ) - ] + assert used_tables == [] From 8d76af8803d76165dfe2ece4571eccfc50be0272 Mon Sep 17 00:00:00 2001 From: pritishpai Date: Mon, 4 Aug 2025 14:07:25 -0400 Subject: [PATCH 05/13] Remove make_dashboard from conftest --- tests/integration/conftest.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2c3e87869d..a0c6ec665e 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -542,12 +542,6 @@ def make_query(self, **kwargs) -> LegacyQuery: self._queries.append(query) return query - def make_dashboard(self, *, query: LegacyQuery | None = None, **kwargs) -> SdkRedashDashboard: - dashboard = self._make_dashboard(query=query, **kwargs) - if query: - self._queries.append(query) - self._dashboards.append(dashboard) - return dashboard def make_lakeview_dashboard(self, **kwargs) -> SdkLakeviewDashboard: dashboard = self._make_lakeview_dashboard(**kwargs) From 96129a492dad60ef55395963c6d90356c5fb6644 Mon Sep 17 00:00:00 2001 From: pritishpai Date: Mon, 4 Aug 2025 14:23:51 -0400 Subject: [PATCH 06/13] test_redash_dashboard_ownership_is_me completely depends on make_dashboard. Removed --- tests/integration/assessment/test_dashboards.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/integration/assessment/test_dashboards.py b/tests/integration/assessment/test_dashboards.py index c57d818570..cc7e250c36 100644 --- a/tests/integration/assessment/test_dashboards.py +++ b/tests/integration/assessment/test_dashboards.py @@ -72,19 +72,6 @@ def test_lakeview_dashboard_crawler_crawls_dashboard( assert dashboards == [Dashboard.from_sdk_lakeview_dashboard(dashboard)] -@pytest.mark.skip( - reason="The user cannot be found using the Dashboard creator user ID when running this test from the CI" -) -def test_redash_dashboard_ownership_is_me(runtime_ctx) -> None: - """The Redash owner should be the user that creates the dashboard, i.e. who runs this integration test.""" - sdk_redash_dashboard = runtime_ctx.make_dashboard() - dashboard = Dashboard.from_sdk_redash_dashboard(sdk_redash_dashboard) - - owner = runtime_ctx.dashboard_ownership.owner_of(dashboard) - - current_user = runtime_ctx.workspace_client.current_user.me() - assert owner == current_user.user_name, f"Invalid owner for dashboard: {dashboard}" - def test_lakeview_dashboard_ownership_is_me(runtime_ctx, make_lakeview_dashboard) -> None: """Lakeview dashboard do not have a `creator` field, thus we fall back on the parent workspace path owner""" From 9fd4ab728e6999a1c65fc01394dcbbf72e36251d Mon Sep 17 00:00:00 2001 From: pritishpai Date: Mon, 4 Aug 2025 15:05:46 -0400 Subject: [PATCH 07/13] remove make_dashboard from runtime_ctx --- tests/integration/conftest.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a0c6ec665e..171af8ca96 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -788,7 +788,6 @@ def runtime_ctx( # pylint: disable=too-many-arguments make_job, make_notebook, make_query, - make_dashboard, make_lakeview_dashboard, make_cluster_policy, make_cluster_policy_permissions, @@ -806,7 +805,6 @@ def runtime_ctx( # pylint: disable=too-many-arguments make_job, make_notebook, make_query, - make_dashboard, make_lakeview_dashboard, make_cluster_policy, make_cluster_policy_permissions, From 6e2661afcc7211c744ad2b42f40c3c8ca837046b Mon Sep 17 00:00:00 2001 From: pritishpai Date: Mon, 4 Aug 2025 15:39:49 -0400 Subject: [PATCH 08/13] remove unused make_dashboard from test_running_real_assessment_job --- tests/integration/assessment/test_workflows.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/assessment/test_workflows.py b/tests/integration/assessment/test_workflows.py index e58a8f509f..c37c0909c4 100644 --- a/tests/integration/assessment/test_workflows.py +++ b/tests/integration/assessment/test_workflows.py @@ -8,7 +8,6 @@ @retried(on=[NotFound, InvalidParameterValue]) def test_running_real_assessment_job( installation_ctx, - make_dashboard, sql_backend, ) -> None: ws_group, _ = installation_ctx.make_ucx_group() From e30705e933fb36ec639e2199d73db977eeb446a5 Mon Sep 17 00:00:00 2001 From: pritishpai Date: Mon, 4 Aug 2025 16:50:44 -0400 Subject: [PATCH 09/13] fmt --- tests/integration/source_code/test_queries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/source_code/test_queries.py b/tests/integration/source_code/test_queries.py index 250e17cb9c..ac74890a88 100644 --- a/tests/integration/source_code/test_queries.py +++ b/tests/integration/source_code/test_queries.py @@ -44,4 +44,4 @@ def test_query_linter_lints_queries_and_stores_dfsas_and_tables(simple_ctx) -> N # By comparing the element instead of the list the `field(compare=False)` of the dataclass attributes take effect # The "query" in the source and object id, and "count" in the name are hardcoded in the # `make_lakeview_dashboard` fixture - assert used_tables == [] + assert not used_tables From 45505102f512f5eefdc63b90167861725ec3d12d Mon Sep 17 00:00:00 2001 From: pritishpai Date: Mon, 4 Aug 2025 20:59:06 -0400 Subject: [PATCH 10/13] remove test_redash_dashboard_crawler_crawls_dashboards_with_debug_listing_upper_limit entirely dependent on legacy dashboards and fmt --- tests/integration/assessment/test_dashboards.py | 14 -------------- tests/integration/conftest.py | 1 - tests/integration/source_code/test_queries.py | 7 ++++--- 3 files changed, 4 insertions(+), 18 deletions(-) diff --git a/tests/integration/assessment/test_dashboards.py b/tests/integration/assessment/test_dashboards.py index cc7e250c36..0eeb936d65 100644 --- a/tests/integration/assessment/test_dashboards.py +++ b/tests/integration/assessment/test_dashboards.py @@ -32,19 +32,6 @@ def test_redash_dashboard_crawler_crawls_dashboard(ws, make_dashboard, inventory assert dashboards == [Dashboard.from_sdk_redash_dashboard(dashboard)] -@pytest.mark.skip(reason="Legacy dashboard creation is no longer supported by Databricks.") -def test_redash_dashboard_crawler_crawls_dashboards_with_debug_listing_upper_limit( - ws, make_dashboard, inventory_schema, sql_backend -) -> None: - for _ in range(2): # Create two dashboards, expect one to be snapshotted due to upper limit below - make_dashboard() - crawler = RedashDashboardCrawler(ws, sql_backend, inventory_schema, debug_listing_upper_limit=1) - - dashboards = list(crawler.snapshot()) - - assert len(dashboards) == 1 - - def test_lakeview_dashboard_crawler_crawls_dashboards( ws, make_lakeview_dashboard, inventory_schema, sql_backend ) -> None: @@ -72,7 +59,6 @@ def test_lakeview_dashboard_crawler_crawls_dashboard( assert dashboards == [Dashboard.from_sdk_lakeview_dashboard(dashboard)] - def test_lakeview_dashboard_ownership_is_me(runtime_ctx, make_lakeview_dashboard) -> None: """Lakeview dashboard do not have a `creator` field, thus we fall back on the parent workspace path owner""" sdk_lakeview_dashboard = make_lakeview_dashboard() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 171af8ca96..31603b04f0 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -542,7 +542,6 @@ def make_query(self, **kwargs) -> LegacyQuery: self._queries.append(query) return query - def make_lakeview_dashboard(self, **kwargs) -> SdkLakeviewDashboard: dashboard = self._make_lakeview_dashboard(**kwargs) self._lakeview_query_id = "query" # Hardcoded query name in the `make_lakeview_dashboard` fixture diff --git a/tests/integration/source_code/test_queries.py b/tests/integration/source_code/test_queries.py index ac74890a88..ff91a6fabd 100644 --- a/tests/integration/source_code/test_queries.py +++ b/tests/integration/source_code/test_queries.py @@ -1,7 +1,7 @@ -import pytest from databricks.labs.lsql.backends import Row -from databricks.labs.ucx.source_code.base import DirectFsAccess, LineageAtom, UsedTable +from databricks.labs.ucx.source_code.base import DirectFsAccess, LineageAtom + def test_query_linter_lints_queries_and_stores_dfsas_and_tables(simple_ctx) -> None: query_with_dfsa = simple_ctx.make_query(sql_query="SELECT * from csv.`dbfs://some_folder/some_file.csv`") @@ -33,7 +33,8 @@ def test_query_linter_lints_queries_and_stores_dfsas_and_tables(simple_ctx) -> N object_type="QUERY", object_id=f"no-dashboard-id/{query_with_dfsa.id}", other={"name": query_with_dfsa.name}, - )], + ) + ], path="dbfs://some_folder/some_file.csv", is_read=True, is_write=False, From 4507819f0d8696f059e5eb737cc3f8116568264a Mon Sep 17 00:00:00 2001 From: pritishpai Date: Mon, 4 Aug 2025 21:00:25 -0400 Subject: [PATCH 11/13] remove test_redash_dashboard_crawler_crawls_dashboards, test_redash_dashboard_crawler_crawls_dashboard dependent on legacy dashboards --- .../integration/assessment/test_dashboards.py | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/tests/integration/assessment/test_dashboards.py b/tests/integration/assessment/test_dashboards.py index 0eeb936d65..e12c2340d6 100644 --- a/tests/integration/assessment/test_dashboards.py +++ b/tests/integration/assessment/test_dashboards.py @@ -9,29 +9,6 @@ ) -@pytest.mark.skip(reason="Legacy dashboard creation is no longer supported by Databricks.") -def test_redash_dashboard_crawler_crawls_dashboards(ws, make_dashboard, inventory_schema, sql_backend) -> None: - dashboard: SdkRedashDashboard = make_dashboard() - crawler = RedashDashboardCrawler(ws, sql_backend, inventory_schema) - - dashboards = list(crawler.snapshot()) - - assert len(dashboards) >= 1 - assert dashboard.id in {d.id for d in dashboards}, f"Missing dashboard: {dashboard.id}" - - -@pytest.mark.skip(reason="Legacy dashboard creation is no longer supported by Databricks.") -def test_redash_dashboard_crawler_crawls_dashboard(ws, make_dashboard, inventory_schema, sql_backend) -> None: - dashboard: SdkRedashDashboard = make_dashboard() - assert dashboard.id - make_dashboard() # Ignore second dashboard - crawler = RedashDashboardCrawler(ws, sql_backend, inventory_schema, include_dashboard_ids=[dashboard.id]) - - dashboards = list(crawler.snapshot()) - - assert dashboards == [Dashboard.from_sdk_redash_dashboard(dashboard)] - - def test_lakeview_dashboard_crawler_crawls_dashboards( ws, make_lakeview_dashboard, inventory_schema, sql_backend ) -> None: From c0794ed5d40229a8ae8b5dfdb2c254d0e7d4ff91 Mon Sep 17 00:00:00 2001 From: pritishpai Date: Mon, 4 Aug 2025 21:38:47 -0400 Subject: [PATCH 12/13] fmt#2 --- tests/integration/assessment/test_dashboards.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/integration/assessment/test_dashboards.py b/tests/integration/assessment/test_dashboards.py index e12c2340d6..fcbcf2a947 100644 --- a/tests/integration/assessment/test_dashboards.py +++ b/tests/integration/assessment/test_dashboards.py @@ -1,11 +1,8 @@ -import pytest -from databricks.sdk.service.sql import Dashboard as SdkRedashDashboard from databricks.sdk.service.dashboards import Dashboard as SdkLakeviewDashboard from databricks.labs.ucx.assessment.dashboards import ( LakeviewDashboardCrawler, Dashboard, - RedashDashboardCrawler, ) From 922fa6501fcf33caa1c8490f109d1254473bd2a1 Mon Sep 17 00:00:00 2001 From: pritishpai Date: Wed, 6 Aug 2025 14:06:38 -0400 Subject: [PATCH 13/13] Bump databricks-sdk to 0.63 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 35693cce9d..6ffc45b1d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ classifiers = [ ] -dependencies = ["databricks-sdk>=0.61.0,<0.62.0", +dependencies = ["databricks-sdk>=0.62.0,<0.63.0", "databricks-labs-lsql>=0.16.0,<0.17.0", "databricks-labs-blueprint>=0.11.0,<0.12.0", "PyYAML>=6.0.0,<6.1.0",