diff --git a/pyproject.toml b/pyproject.toml index 5cff256b28..6ffc45b1d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ classifiers = [ ] -dependencies = ["databricks-sdk>=0.58.0,<0.59.0", +dependencies = ["databricks-sdk>=0.62.0,<0.63.0", "databricks-labs-lsql>=0.16.0,<0.17.0", "databricks-labs-blueprint>=0.11.0,<0.12.0", "PyYAML>=6.0.0,<6.1.0", diff --git a/tests/integration/assessment/test_dashboards.py b/tests/integration/assessment/test_dashboards.py index c57d818570..fcbcf2a947 100644 --- a/tests/integration/assessment/test_dashboards.py +++ b/tests/integration/assessment/test_dashboards.py @@ -1,50 +1,11 @@ -import pytest -from databricks.sdk.service.sql import Dashboard as SdkRedashDashboard from databricks.sdk.service.dashboards import Dashboard as SdkLakeviewDashboard from databricks.labs.ucx.assessment.dashboards import ( LakeviewDashboardCrawler, Dashboard, - RedashDashboardCrawler, ) -@pytest.mark.skip(reason="Legacy dashboard creation is no longer supported by Databricks.") -def test_redash_dashboard_crawler_crawls_dashboards(ws, make_dashboard, inventory_schema, sql_backend) -> None: - dashboard: SdkRedashDashboard = make_dashboard() - crawler = RedashDashboardCrawler(ws, sql_backend, inventory_schema) - - dashboards = list(crawler.snapshot()) - - assert len(dashboards) >= 1 - assert dashboard.id in {d.id for d in dashboards}, f"Missing dashboard: {dashboard.id}" - - -@pytest.mark.skip(reason="Legacy dashboard creation is no longer supported by Databricks.") -def test_redash_dashboard_crawler_crawls_dashboard(ws, make_dashboard, inventory_schema, sql_backend) -> None: - dashboard: SdkRedashDashboard = make_dashboard() - assert dashboard.id - make_dashboard() # Ignore second dashboard - crawler = RedashDashboardCrawler(ws, sql_backend, inventory_schema, include_dashboard_ids=[dashboard.id]) - - dashboards = list(crawler.snapshot()) - - assert dashboards == [Dashboard.from_sdk_redash_dashboard(dashboard)] - - -@pytest.mark.skip(reason="Legacy dashboard creation is no longer supported by Databricks.") -def test_redash_dashboard_crawler_crawls_dashboards_with_debug_listing_upper_limit( - ws, make_dashboard, inventory_schema, sql_backend -) -> None: - for _ in range(2): # Create two dashboards, expect one to be snapshotted due to upper limit below - make_dashboard() - crawler = RedashDashboardCrawler(ws, sql_backend, inventory_schema, debug_listing_upper_limit=1) - - dashboards = list(crawler.snapshot()) - - assert len(dashboards) == 1 - - def test_lakeview_dashboard_crawler_crawls_dashboards( ws, make_lakeview_dashboard, inventory_schema, sql_backend ) -> None: @@ -72,20 +33,6 @@ def test_lakeview_dashboard_crawler_crawls_dashboard( assert dashboards == [Dashboard.from_sdk_lakeview_dashboard(dashboard)] -@pytest.mark.skip( - reason="The user cannot be found using the Dashboard creator user ID when running this test from the CI" -) -def test_redash_dashboard_ownership_is_me(runtime_ctx) -> None: - """The Redash owner should be the user that creates the dashboard, i.e. who runs this integration test.""" - sdk_redash_dashboard = runtime_ctx.make_dashboard() - dashboard = Dashboard.from_sdk_redash_dashboard(sdk_redash_dashboard) - - owner = runtime_ctx.dashboard_ownership.owner_of(dashboard) - - current_user = runtime_ctx.workspace_client.current_user.me() - assert owner == current_user.user_name, f"Invalid owner for dashboard: {dashboard}" - - def test_lakeview_dashboard_ownership_is_me(runtime_ctx, make_lakeview_dashboard) -> None: """Lakeview dashboard do not have a `creator` field, thus we fall back on the parent workspace path owner""" sdk_lakeview_dashboard = make_lakeview_dashboard() diff --git a/tests/integration/assessment/test_workflows.py b/tests/integration/assessment/test_workflows.py index e58a8f509f..c37c0909c4 100644 --- a/tests/integration/assessment/test_workflows.py +++ b/tests/integration/assessment/test_workflows.py @@ -8,7 +8,6 @@ @retried(on=[NotFound, InvalidParameterValue]) def test_running_real_assessment_job( installation_ctx, - make_dashboard, sql_backend, ) -> None: ws_group, _ = installation_ctx.make_ucx_group() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1ba4478dd9..31603b04f0 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -29,7 +29,7 @@ from databricks.sdk.service.dashboards import Dashboard as SdkLakeviewDashboard from databricks.sdk.service.iam import Group from databricks.sdk.service.jobs import Job, SparkPythonTask -from databricks.sdk.service.sql import Dashboard as SdkRedashDashboard, WidgetPosition, WidgetOptions, LegacyQuery +from databricks.sdk.service.sql import Dashboard as SdkRedashDashboard, LegacyQuery from databricks.labs.ucx.__about__ import __version__ from databricks.labs.ucx.account.workspaces import AccountWorkspaces @@ -135,66 +135,6 @@ def delete(dashboard: SdkLakeviewDashboard) -> None: yield from factory("dashboard", create, delete) -@pytest.fixture -def make_dashboard( - ws: WorkspaceClient, - make_random: Callable[[int], str], - make_query, - watchdog_purge_suffix, -): - """Create a legacy dashboard. - This fixture is used to test migrating legacy dashboards to Lakeview. - """ - - def create(query: LegacyQuery | None = None) -> SdkRedashDashboard: - if not query: - query = make_query() - assert query - assert query.id - viz = ws.query_visualizations_legacy.create( - type="table", - query_id=query.id, - options={ - "itemsPerPage": 1, - "condensed": True, - "withRowNumber": False, - "version": 2, - "columns": [ - {"name": "id", "title": "id", "allowSearch": True}, - ], - }, - ) - - dashboard_name = f"ucx_D{make_random(4)}_{watchdog_purge_suffix}" - dashboard = ws.dashboards.create(name=dashboard_name, tags=["original_dashboard_tag"]) - assert dashboard.id is not None - ws.dashboard_widgets.create( - dashboard_id=dashboard.id, - visualization_id=viz.id, - width=1, - options=WidgetOptions( - title="", - position=WidgetPosition( - col=0, - row=0, - size_x=3, - size_y=3, - ), - ), - ) - logger.info(f"Dashboard Created {dashboard_name}: {ws.config.host}/sql/dashboards/{dashboard.id}") - return ws.dashboards.get(dashboard.id) # Dashboard with widget - - def remove(dashboard: SdkRedashDashboard) -> None: - try: - assert dashboard.id is not None - ws.dashboards.delete(dashboard_id=dashboard.id) - except RuntimeError as e: - logger.info(f"Can't delete dashboard {e}") - - yield from factory("dashboard", create, remove) - - @pytest.fixture def make_dbfs_data_copy(ws, make_cluster, env_or_skip): _ = make_cluster # Need cluster to copy data @@ -467,7 +407,6 @@ def __init__( # pylint: disable=too-many-arguments make_job_fixture, make_notebook_fixture, make_query_fixture, - make_dashboard_fixture, make_lakeview_dashboard_fixture, make_cluster_policy_fixture, make_cluster_policy_permissions_fixture, @@ -492,7 +431,6 @@ def __init__( # pylint: disable=too-many-arguments self._make_job = make_job_fixture self._make_notebook = make_notebook_fixture self._make_query = make_query_fixture - self._make_dashboard = make_dashboard_fixture self._make_lakeview_dashboard = make_lakeview_dashboard_fixture self._make_cluster_policy = make_cluster_policy_fixture self._make_cluster_policy_permissions = make_cluster_policy_permissions_fixture @@ -604,13 +542,6 @@ def make_query(self, **kwargs) -> LegacyQuery: self._queries.append(query) return query - def make_dashboard(self, *, query: LegacyQuery | None = None, **kwargs) -> SdkRedashDashboard: - dashboard = self._make_dashboard(query=query, **kwargs) - if query: - self._queries.append(query) - self._dashboards.append(dashboard) - return dashboard - def make_lakeview_dashboard(self, **kwargs) -> SdkLakeviewDashboard: dashboard = self._make_lakeview_dashboard(**kwargs) self._lakeview_query_id = "query" # Hardcoded query name in the `make_lakeview_dashboard` fixture @@ -856,7 +787,6 @@ def runtime_ctx( # pylint: disable=too-many-arguments make_job, make_notebook, make_query, - make_dashboard, make_lakeview_dashboard, make_cluster_policy, make_cluster_policy_permissions, @@ -874,7 +804,6 @@ def runtime_ctx( # pylint: disable=too-many-arguments make_job, make_notebook, make_query, - make_dashboard, make_lakeview_dashboard, make_cluster_policy, make_cluster_policy_permissions, @@ -1018,7 +947,6 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals make_job_fixture, make_notebook_fixture, make_query_fixture, - make_dashboard_fixture, make_lakeview_dashboard_fixture, make_cluster_policy, make_cluster_policy_permissions, @@ -1036,7 +964,6 @@ def __init__( # pylint: disable=too-many-arguments, too-many-locals make_job_fixture, make_notebook_fixture, make_query_fixture, - make_dashboard_fixture, make_lakeview_dashboard_fixture, make_cluster_policy, make_cluster_policy_permissions, @@ -1218,7 +1145,6 @@ def installation_ctx( # pylint: disable=too-many-arguments,too-many-locals make_job, make_notebook, make_query, - make_dashboard, make_lakeview_dashboard, make_cluster_policy, make_cluster_policy_permissions, @@ -1239,7 +1165,6 @@ def installation_ctx( # pylint: disable=too-many-arguments,too-many-locals make_job, make_notebook, make_query, - make_dashboard, make_lakeview_dashboard, make_cluster_policy, make_cluster_policy_permissions, diff --git a/tests/integration/source_code/test_queries.py b/tests/integration/source_code/test_queries.py index e993d49e5d..ff91a6fabd 100644 --- a/tests/integration/source_code/test_queries.py +++ b/tests/integration/source_code/test_queries.py @@ -1,24 +1,20 @@ -import pytest from databricks.labs.lsql.backends import Row -from databricks.labs.ucx.source_code.base import DirectFsAccess, LineageAtom, UsedTable +from databricks.labs.ucx.source_code.base import DirectFsAccess, LineageAtom -@pytest.mark.skip(reason="Legacy dashboard creation is no longer supported by Databricks.") def test_query_linter_lints_queries_and_stores_dfsas_and_tables(simple_ctx) -> None: query_with_dfsa = simple_ctx.make_query(sql_query="SELECT * from csv.`dbfs://some_folder/some_file.csv`") - dashboard_with_dfsa = simple_ctx.make_dashboard(query=query_with_dfsa) # Lakeview dashboard expects a string, not a legacy query - dashboard_with_used_table = simple_ctx.make_lakeview_dashboard(query="SELECT * FROM some_schema.some_table") simple_ctx.query_linter.refresh_report() problems = list(simple_ctx.sql_backend.fetch("SELECT * FROM query_problems", schema=simple_ctx.inventory_database)) assert problems == [ Row( - dashboard_id=dashboard_with_dfsa.id, - dashboard_parent=dashboard_with_dfsa.parent, - dashboard_name=dashboard_with_dfsa.name, + dashboard_id=None, + dashboard_parent=None, + dashboard_name=None, query_id=query_with_dfsa.id, query_parent=query_with_dfsa.parent, query_name=query_with_dfsa.name, @@ -31,18 +27,13 @@ def test_query_linter_lints_queries_and_stores_dfsas_and_tables(simple_ctx) -> N # By comparing the element instead of the list the `field(compare=False)` of the dataclass attributes take effect assert dfsas == [ DirectFsAccess( - source_id=f"{dashboard_with_dfsa.id}/{query_with_dfsa.id}", + source_id=f"no-dashboard-id/{query_with_dfsa.id}", source_lineage=[ - LineageAtom( - object_type="DASHBOARD", - object_id=dashboard_with_dfsa.id, - other={"parent": dashboard_with_dfsa.parent, "name": dashboard_with_dfsa.name}, - ), LineageAtom( object_type="QUERY", - object_id=f"{dashboard_with_dfsa.id}/{query_with_dfsa.id}", + object_id=f"no-dashboard-id/{query_with_dfsa.id}", other={"name": query_with_dfsa.name}, - ), + ) ], path="dbfs://some_folder/some_file.csv", is_read=True, @@ -54,28 +45,4 @@ def test_query_linter_lints_queries_and_stores_dfsas_and_tables(simple_ctx) -> N # By comparing the element instead of the list the `field(compare=False)` of the dataclass attributes take effect # The "query" in the source and object id, and "count" in the name are hardcoded in the # `make_lakeview_dashboard` fixture - assert used_tables == [ - UsedTable( - source_id=f"{dashboard_with_used_table.dashboard_id}/query", - source_lineage=[ - LineageAtom( - object_type="DASHBOARD", - object_id=dashboard_with_used_table.dashboard_id, - other={ - "parent": dashboard_with_used_table.parent_path, - "name": dashboard_with_used_table.display_name, - }, - ), - LineageAtom( - object_type="QUERY", - object_id=f"{dashboard_with_used_table.dashboard_id}/query", - other={"name": "count"}, - ), - ], - catalog_name="hive_metastore", - schema_name="some_schema", - table_name="some_table", - is_read=True, - is_write=False, - ) - ] + assert not used_tables