Skip to content

Commit 55df7fb

Browse files
AdityaMandiwalmwojtyczka
AdityaMandiwal
and
mwojtyczka
authored
Updated dashboards deployment to use standard lakeview dashboard definitions (#950)
## Changes <!-- updated dashboards to use standard Lakeview dashboards definitions in place of lsql library --> * Updated dashboards deployment to use standard lakeview dashboard definitions. Removed `lsql` dependency. * Improved telemetry by logging unique hashed inputs ### Linked issues <!-- DOC: Link issue with a keyword: close, closes, closed, fix, fixes, fixed, resolve, resolves, resolved. See https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword --> Resolves #132 ### Tests <!-- How is this tested? Please see the checklist below and also describe any other relevant tests --> - [x] manually tested - [ ] added unit tests - [ ] added integration tests - [ ] added end-to-end tests - [ ] added performance tests --------- Co-authored-by: Marcin Wojtyczka <marcin.wojtyczka@databricks.com>
1 parent d200468 commit 55df7fb

File tree

13 files changed

+828
-295
lines changed

13 files changed

+828
-295
lines changed

‎pyproject.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,9 @@ classifiers = [
3737
# also update fallback dependencies in workflow installer when changing dependencies!
3838
dependencies = [
3939
"databricks-labs-blueprint>=0.9.1,<0.10",
40-
"databricks-sdk~=0.71",
41-
"databricks-labs-lsql>=0.5,<=0.16",
40+
"databricks-sdk~=0.73",
4241
"sqlalchemy>=2.0,<3.0",
42+
"PyYAML~=6.0.3",
4343
]
4444

4545
[project.optional-dependencies]
@@ -111,7 +111,7 @@ dependencies = [
111111
"pytest-xdist~=3.5.0",
112112
"pytest-benchmark~=5.1.0",
113113
"ruff~=0.3.4",
114-
"types-PyYAML~=6.0.12",
114+
"types-PyYAML~=6.0.3",
115115
"types-requests~=2.31.0",
116116
"databricks-connect~=15.4",
117117
"pyspark~=3.5.0",

‎src/databricks/labs/dqx/dashboards/dashboard.lvdash.json‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
{
2+
"datasets": [
3+
{
4+
"name": "00_1_dq_summary",
5+
"displayName": "00_1_dq_summary",
6+
"queryLines": [
7+
"/* --title 'Data Quality Summary' */\n",
8+
"SELECT \n",
9+
" CASE\n",
10+
" WHEN _errors IS NOT NULL THEN 'Errors'\n",
11+
" WHEN _warnings IS NOT NULL THEN 'Warnings'\n",
12+
" END AS category,\n",
13+
" ROUND((COUNT(*) * 100.0 / (SELECT COUNT(*) FROM $catalog.schema.table)), 2) AS percentage\n",
14+
"FROM $catalog.schema.table\n",
15+
"GROUP BY category"
16+
]
17+
},
18+
{
19+
"name": "00_2_dq_error_types",
20+
"displayName": "00_2_dq_error_types",
21+
"queryLines": [
22+
"/* --title 'Error and Warning Types Breakdown' */\n",
23+
"WITH error_types AS (\n",
24+
" SELECT\n",
25+
" 'Error' AS category,\n",
26+
" error_struct.name AS type,\n",
27+
" COUNT(*) AS count\n",
28+
" FROM $catalog.schema.table\n",
29+
" LATERAL VIEW EXPLODE(_errors) exploded_errors AS error_struct\n",
30+
" WHERE _errors IS NOT NULL\n",
31+
" GROUP BY error_struct.name\n",
32+
"),\n",
33+
"warning_types AS (\n",
34+
" SELECT\n",
35+
" 'Warning' AS category,\n",
36+
" warning_struct.name AS type,\n",
37+
" COUNT(*) AS count\n",
38+
" FROM $catalog.schema.table\n",
39+
" LATERAL VIEW EXPLODE(_warnings) exploded_warnings AS warning_struct\n",
40+
" WHERE _warnings IS NOT NULL\n",
41+
" GROUP BY warning_struct.name\n",
42+
"),\n",
43+
"combined AS (\n",
44+
" SELECT * FROM error_types\n",
45+
" UNION ALL\n",
46+
" SELECT * FROM warning_types\n",
47+
"),\n",
48+
"total AS (\n",
49+
" SELECT SUM(count) AS total_count FROM combined\n",
50+
")\n",
51+
"SELECT\n",
52+
" category,\n",
53+
" type,\n",
54+
" count,\n",
55+
" ROUND((count * 100.0) / total.total_count, 2) AS percentage\n",
56+
"FROM combined, total\n",
57+
"ORDER BY category, count DESC;"
58+
]
59+
}
60+
],
61+
"pages": [
62+
{
63+
"name": "DQX_Quality_Dashboard",
64+
"displayName": "DQX_Quality_Dashboard",
65+
"layout": [
66+
{
67+
"widget": {
68+
"name": "00_1_dq_summary_widget",
69+
"queries": [
70+
{
71+
"name": "main_query",
72+
"query": {
73+
"datasetName": "00_1_dq_summary",
74+
"fields": [
75+
{
76+
"name": "sum(percentage)",
77+
"expression": "SUM(`percentage`)"
78+
},
79+
{
80+
"name": "category",
81+
"expression": "`category`"
82+
}
83+
],
84+
"disaggregated": false
85+
}
86+
}
87+
],
88+
"spec": {
89+
"version": 3,
90+
"widgetType": "pie",
91+
"encodings": {
92+
"angle": {
93+
"displayName": "Total Row Counts",
94+
"fieldName": "sum(percentage)",
95+
"scale": {
96+
"type": "quantitative"
97+
}
98+
},
99+
"color": {
100+
"displayName": "category",
101+
"fieldName": "category",
102+
"scale": {
103+
"type": "categorical",
104+
"mappings": [
105+
{
106+
"color": "#00A972"
107+
},
108+
{
109+
"color": "#FFAB00"
110+
},
111+
{
112+
"color": "#FF3621"
113+
}
114+
]
115+
}
116+
},
117+
"label": {
118+
"show": true
119+
}
120+
},
121+
"frame": {
122+
"description": "",
123+
"showDescription": false,
124+
"showTitle": true,
125+
"title": "Data Quality Summary"
126+
}
127+
}
128+
},
129+
"position": {
130+
"x": 0,
131+
"y": 0,
132+
"width": 3,
133+
"height": 6
134+
}
135+
},
136+
{
137+
"widget": {
138+
"name": "00_2_dq_error_types_widget",
139+
"queries": [
140+
{
141+
"name": "main_query",
142+
"query": {
143+
"datasetName": "00_2_dq_error_types",
144+
"fields": [
145+
{
146+
"name": "Category",
147+
"expression": "`category`"
148+
},
149+
{
150+
"name": "Type",
151+
"expression": "`type`"
152+
},
153+
{
154+
"name": "Count",
155+
"expression": "`count`"
156+
},
157+
{
158+
"name": "Percentage",
159+
"expression": "`percentage`"
160+
}
161+
],
162+
"disaggregated": true
163+
}
164+
}
165+
],
166+
"spec": {
167+
"version": 2,
168+
"widgetType": "table",
169+
"encodings": {
170+
"columns": [
171+
{
172+
"displayName": "Category",
173+
"fieldName": "Category"
174+
},
175+
{
176+
"displayName": "Type",
177+
"fieldName": "Type"
178+
},
179+
{
180+
"displayName": "Count",
181+
"fieldName": "Count"
182+
},
183+
{
184+
"displayName": "Percentage (%)",
185+
"fieldName": "Percentage"
186+
}
187+
]
188+
},
189+
"frame": {
190+
"description": "",
191+
"showDescription": false,
192+
"showTitle": true,
193+
"title": "Error and Warning Types Breakdown"
194+
}
195+
}
196+
},
197+
"position": {
198+
"x": 3,
199+
"y": 0,
200+
"width": 3,
201+
"height": 6
202+
}
203+
}
204+
],
205+
"pageType": "PAGE_TYPE_CANVAS"
206+
}
207+
]
208+
}

0 commit comments

Comments
 (0)