From 364e750ce912138347df0f3d3fbf34142b68ccd1 Mon Sep 17 00:00:00 2001 From: matthias Date: Tue, 3 Dec 2024 11:20:39 +0100 Subject: [PATCH 01/32] add new pgwatch-helm-chart for k8s/openshift --- helm/pgwatch/Chart.yaml | 5 + .../templates/grafana-dashboards-basics.yaml | 18463 +++++++++++++ .../pgwatch/templates/grafana-deployment.yaml | 118 + .../templates/grafana_dashboard_others.yaml | 22971 ++++++++++++++++ .../pgwatch/templates/pgwatch-deployment.yaml | 44 + .../templates/postgres-statefulset.yaml | 79 + helm/pgwatch/templates/secrets.yaml | 19 + helm/pgwatch/templates/services.yaml | 44 + helm/pgwatch/values.yaml | 8 + helm/{ => pgwatch2(old)}/.helmignore | 0 helm/{ => pgwatch2(old)}/Chart.lock | 0 helm/{ => pgwatch2(old)}/Chart.yaml | 0 helm/{ => pgwatch2(old)}/README.md | 0 .../charts/influxdb-4.12.0.tgz | Bin .../charts/timescaledb-single-0.33.1.tgz | Bin .../grafana-6.50.0.tgz | Bin helm/{ => pgwatch2(old)}/grafana_dashboards | 0 helm/{ => pgwatch2(old)}/templates/NOTES.txt | 0 .../templates/_helpers.tpl | 0 .../templates/configmaps.yaml | 0 .../templates/configmaps2.yaml | 0 .../templates/deployment.yaml | 0 .../templates/grafana-dashboards.yaml | 0 .../templates/ingress.yaml | 0 .../templates/service.yaml | 0 helm/{ => pgwatch2(old)}/values.yaml | 0 26 files changed, 41751 insertions(+) create mode 100644 helm/pgwatch/Chart.yaml create mode 100644 helm/pgwatch/templates/grafana-dashboards-basics.yaml create mode 100644 helm/pgwatch/templates/grafana-deployment.yaml create mode 100644 helm/pgwatch/templates/grafana_dashboard_others.yaml create mode 100644 helm/pgwatch/templates/pgwatch-deployment.yaml create mode 100644 helm/pgwatch/templates/postgres-statefulset.yaml create mode 100644 helm/pgwatch/templates/secrets.yaml create mode 100644 helm/pgwatch/templates/services.yaml create mode 100644 helm/pgwatch/values.yaml rename helm/{ => pgwatch2(old)}/.helmignore (100%) rename helm/{ => pgwatch2(old)}/Chart.lock (100%) rename helm/{ => pgwatch2(old)}/Chart.yaml (100%) rename helm/{ => pgwatch2(old)}/README.md (100%) rename helm/{ => pgwatch2(old)}/charts/influxdb-4.12.0.tgz (100%) rename helm/{ => pgwatch2(old)}/charts/timescaledb-single-0.33.1.tgz (100%) rename helm/{charts => pgwatch2(old)}/grafana-6.50.0.tgz (100%) rename helm/{ => pgwatch2(old)}/grafana_dashboards (100%) rename helm/{ => pgwatch2(old)}/templates/NOTES.txt (100%) rename helm/{ => pgwatch2(old)}/templates/_helpers.tpl (100%) rename helm/{ => pgwatch2(old)}/templates/configmaps.yaml (100%) rename helm/{ => pgwatch2(old)}/templates/configmaps2.yaml (100%) rename helm/{ => pgwatch2(old)}/templates/deployment.yaml (100%) rename helm/{ => pgwatch2(old)}/templates/grafana-dashboards.yaml (100%) rename helm/{ => pgwatch2(old)}/templates/ingress.yaml (100%) rename helm/{ => pgwatch2(old)}/templates/service.yaml (100%) rename helm/{ => pgwatch2(old)}/values.yaml (100%) diff --git a/helm/pgwatch/Chart.yaml b/helm/pgwatch/Chart.yaml new file mode 100644 index 0000000..9faa66b --- /dev/null +++ b/helm/pgwatch/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "3.0" +description: A Helm chart for pgwatch monitoring tools +name: pgwatch +version: 3.0.0 diff --git a/helm/pgwatch/templates/grafana-dashboards-basics.yaml b/helm/pgwatch/templates/grafana-dashboards-basics.yaml new file mode 100644 index 0000000..4bd4329 --- /dev/null +++ b/helm/pgwatch/templates/grafana-dashboards-basics.yaml @@ -0,0 +1,18463 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + application: pgwatch + name: grafana-dashboards-basics +data: + db-overview-developer.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Only \"pg_stat_statements\" extension expected", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "description": "Last 5m data", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 0 + }, + "hideTimeOverride": true, + "id": 1, + "interval": "", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "alias": "TPS", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time,\n (data->'in_recovery_int')::int\nFROM\n wal\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nORDER BY time DESC\nLIMIT 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "5m", + "title": "Instance state", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "PRIMARY", + "value": "0" + }, + { + "op": "=", + "text": "REPLICA", + "value": "1" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "description": "Last 5min data", + "editable": true, + "error": false, + "format": "dtdurations", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 3, + "y": 0 + }, + "hideTimeOverride": true, + "id": 17, + "interval": "", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "alias": "TPS", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time,\n (data->'postmaster_uptime_s')::int\nFROM\n wal\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nORDER BY time DESC\nLIMIT 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "5m", + "title": "Instance uptime", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "description": "Current Transaction Per Second (10m avg) + last 6h history", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 6, + "y": 0 + }, + "hideTimeOverride": true, + "id": 18, + "interval": "10m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "TPS", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $__interval),\n avg(((c - c_lag) + (r - r_lag)) / extract(epoch from time - time_lag)) as tps\nfrom (\n select \n (data->>'xact_commit')::int8 as c, lag((data->>'xact_commit')::int8) over w as c_lag,\n (data->>'xact_rollback')::int8 as r, lag((data->>'xact_rollback')::int8) over w as r_lag,\n time, lag(time) over w as time_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and r >= r_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "6h", + "title": "TPS", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "description": "Queries Per Second based on pg_stat_statements.calls. Last 10m avg. + last 6h history", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 9, + "y": 0 + }, + "height": "150", + "hideTimeOverride": true, + "id": 2, + "interval": "10m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "QPS", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $__interval),\n avg((c - c_lag) / extract(epoch from time - time_lag)) as qps\nfrom (\n select \n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from stat_statements_calls\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag\ngroup by 1\norder by 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "6h", + "title": "QPS", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "description": "Approximate value based on pg_stat_statements total_time / calls. Last 10min avg. + last 6h history", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 12, + "y": 0 + }, + "height": "150", + "hideTimeOverride": true, + "id": 3, + "interval": "10m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": " ms", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "avg_query_runtime_per_db", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n$__timeGroup(time, $__interval),\navg((tt-tt_lag)::numeric / (c-c_lag))\n\nfrom (\n\n select \n (data->>'total_time')::float8 as tt, lag((data->>'total_time')::float8) over w as tt_lag,\n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n time\n from stat_statements_calls\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere c > c_lag and tt >= tt_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "6h", + "timeShift": null, + "title": "Query runtime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": 1, + "description": "Based on pg_stat_database.numbackends. Last 10m avg + 6h history", + "editable": true, + "error": false, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 15, + "y": 0 + }, + "height": "150", + "hideTimeOverride": true, + "id": 5, + "interval": "10m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "10m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $__interval),\n avg(nb) as \"# Sessions ($agg_interval avg)\"\nfrom (\n select \n (data->>'numbackends')::int8 as nb,\n time\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n) x\ngroup by 1\norder by 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "numbackends" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "6h", + "timeShift": null, + "title": "Sessions", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": null, + "description": "", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 18, + "y": 0 + }, + "height": "150", + "hideTimeOverride": true, + "id": 6, + "interval": "10m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "10m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, '1h'),\n avg( ((tb-tb_lag)*3600) / (etime - lag_etime)) as \"Temp bytes (1h rate)\"\nfrom (\n select \n (data->>'temp_bytes')::int8 as tb, lag((data->>'temp_bytes')::int8) over w as tb_lag,\n (extract(epoch from time))::int8 as etime,\n (lag(extract(epoch from time)) over w)::int8 as lag_etime,\n time\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere tb >= tb_lag and etime > lag_etime\ngroup by 1\norder by 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "temp_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "1h" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "6h", + "timeShift": null, + "title": "Temp bytes 1h avg. rate", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": 1, + "description": "Percentage of rows touched by queries that are actually returned to the client. Low numbers (< 10%) for an OLTP workload can hint at missing indexes. Last 10m avg + 6h history.", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 21, + "y": 0 + }, + "height": "150", + "hideTimeOverride": true, + "id": 4, + "interval": "10m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "db_size_change_last_hour", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $__interval),\n 100 * avg((fetched - fetched_lag)::numeric / (returned - returned_lag)) as fetched_vs_returned_pct\nFROM (\n\nSELECT\n time,\n (data->'tup_fetched')::int8 as fetched,\n lag((data->'tup_fetched')::int8) over w as fetched_lag,\n (data->'tup_returned')::int8 as returned,\n lag((data->'tup_returned')::int8) over w as returned_lag\nFROM\n db_stats\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nWINDOW w AS (ORDER BY time)\n) x \nWHERE returned > returned_lag AND fetched >= fetched_lag\nGROUP BY 1\nORDER BY 1\n\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "6h", + "timeShift": null, + "title": "Tuples fetched vs returned", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 2 + }, + "id": 7, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "db_stats.mean", + "yaxis": 2 + }, + { + "alias": "DB size", + "yaxis": 2 + }, + { + "alias": "blk_read_time", + "yaxis": 2 + }, + { + "alias": "blk_write_time", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "INSERT", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((ins-ins_lag) * (extract(epoch from '$agg_interval'::interval)) / extract(epoch from time - time_lag)) as \"INSERT\",\n avg((upd-upd_lag) * (extract(epoch from '$agg_interval'::interval)) / extract(epoch from time - time_lag)) as \"UPDATE\",\n avg((del-del_lag) * (extract(epoch from '$agg_interval'::interval)) / extract(epoch from time - time_lag)) as \"DELETE\"\nfrom (\n select \n (data->>'tup_inserted')::int8 as ins, lag((data->>'tup_inserted')::int8) over w as ins_lag,\n (data->>'tup_updated')::int8 as upd, lag((data->>'tup_updated')::int8) over w as upd_lag,\n (data->>'tup_deleted')::int8 as del, lag((data->>'tup_deleted')::int8) over w as del_lag,\n time, lag(time) over w as time_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere ins >= ins_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Tuple Ins. / Upd. / Del. statistics ($agg_interval rate)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "TX rollback ratio": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 2 + }, + "id": 8, + "interval": "5m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Shared buffers hit ratio", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((hit-hit_lag)::numeric / ((hit-hit_lag) + (read-read_lag)))*100 as \"Shared buffers hit ratio\",\n avg((roll-roll_lag)::numeric / ((roll-roll_lag) + (comm-comm_lag)))*100 as \"TX rollback ratio\"\nfrom (\n select \n (data->>'blks_hit')::int8 as hit, lag((data->>'blks_hit')::int8) over w as hit_lag,\n (data->>'blks_read')::int8 as read, lag((data->>'blks_read')::int8) over w as read_lag,\n (data->>'xact_rollback')::int8 as roll, lag((data->>'xact_rollback')::int8) over w as roll_lag,\n (data->>'xact_commit')::int8 as comm, lag((data->>'xact_commit')::int8) over w as comm_lag,\n time\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere (hit > hit_lag or read > read_lag) and (roll > roll_lag or comm > comm_lag)\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "blks_hit" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Buffer hit ratio + Rollback ratio", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": "105", + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "avg_query_runtime": "#ef843c", + "load_5": "#BA43A9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "Approximate value based on pg_stat_statements total_time / calls", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 12, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "avg_query_runtime", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "stat_statements", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(case when c=c_lag then null else (tt-tt_lag)::numeric / (c-c_lag) end) as \"avg_query_runtime\"\nfrom (\n\n select \n (data->>'total_time')::float8 as tt, lag((data->>'total_time')::float8) over w as tt_lag,\n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n time\n from stat_statements_calls\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere c >= c_lag and tt >= tt_lag\ngroup by 1\norder by 1", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Avg. query runtime ($agg_interval avg.)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "DB Size": "#2F575E", + "WAL rate": "#0A50A1" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 6 + }, + "height": "", + "id": 10, + "interval": "5m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "DB Size", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "WAL rate", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "wal", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((wal-wal_lag) / (etime-etime_lag)) as \"WAL rate\"\nfrom (\n select \n (data->>'xlog_location_b')::int8 as wal, lag((data->>'xlog_location_b')::int8) over w as wal_lag,\n extract(epoch from time) as etime,\n time, lag(extract(epoch from time)) over w as etime_lag\n from wal\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere wal >= wal_lag and etime > etime_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "xlog_location_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1h" + ], + "type": "derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "alias": "DB Size", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_size", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(size_b) as \"DB Size\"\nfrom (\n select \n (data->>'size_b')::int8 as size_b,\n time\n from db_size\n where dbname = '$dbname' and $__timeFilter(time)\n) x\ngroup by 1\norder by 1", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "size_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "WAL rate + DB size ($agg_interval avg.)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "/Deadlocks/": "dark-red", + "/Sessions/": "dark-yellow", + "/Temp bytes/": "light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 9, + "interval": "5m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Temp bytes/", + "color": "#3274D9", + "yaxis": 2 + }, + { + "alias": "/Deadlocks/", + "color": "#C4162A" + }, + { + "alias": "/Sessions/", + "color": "#E0B400" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "# Sessions", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(nb) as \"# Sessions ($agg_interval avg)\",\n avg(((dl-dl_lag) * extract(epoch from '$agg_interval'::interval)) / (etime - etime_lag)) as \"Deadlocks ($agg_interval rate)\"\nfrom (\n select \n (data->>'numbackends')::int8 as nb,\n (data->>'deadlocks')::int8 as dl, lag((data->>'deadlocks')::int8) over w as dl_lag,\n (extract(epoch from time))::int8 as etime,\n (lag(extract(epoch from time)) over w)::int8 as etime_lag,\n time\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere dl >= dl_lag and etime > etime_lag\ngroup by 1\norder by 1\n", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "numbackends" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "alias": "Deadlocks (1h rate)", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((tb-tb_lag) / (etime - lag_etime)) as \"Temp bytes ($agg_interval avg)\"\nfrom (\n select \n (data->>'temp_bytes')::int8 as tb, lag((data->>'temp_bytes')::int8) over w as tb_lag,\n (extract(epoch from time))::int8 as etime,\n (lag(extract(epoch from time)) over w)::int8 as lag_etime,\n time\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere tb >= tb_lag\ngroup by 1\norder by 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sessions + Deadlocks + Temp bytes", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "Transactions per Second / Queries per Second", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 15, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "TPS", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(((c - c_lag) + (r - r_lag)) / extract(epoch from time - time_lag)) as \"TPS\"\nfrom (\n select \n (data->>'xact_commit')::int8 as c, lag((data->>'xact_commit')::int8) over w as c_lag,\n (data->>'xact_rollback')::int8 as r, lag((data->>'xact_rollback')::int8) over w as r_lag,\n time, lag(time) over w as time_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and r >= r_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "alias": "QPS", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((c - c_lag) / extract(epoch from time - time_lag)) as \"QPS\"\nfrom (\n select \n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from stat_statements_calls\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and time > time_lag\ngroup by 1\norder by 1\n", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TPS / QPS ($agg_interval avg.)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 16, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "seq_scan", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n time,\n sum(seq_scan) as seq_scan\nfrom (\n select\n $__timeGroup(time, $agg_interval),\n table_full_name,\n avg((((scan - scan_lag) * (extract(epoch from '$agg_interval'::interval))) / extract(epoch from time - time_lag))) as seq_scan\n from (\n select \n (data->>'seq_scan')::int8 as scan, lag((data->>'seq_scan')::int8) over w as scan_lag,\n time, lag(time) over w as time_lag,\n tag_data->>'table_full_name' as table_full_name\n from table_stats\n where dbname = '$dbname' and $__timeFilter(time)\n and (data->>'table_size_b')::int8 > 10000000\n window w as (partition by tag_data->>'table_full_name' order by time) \n ) x\n where scan >= scan_lag and time > time_lag\n group by 1, 2\n) y\ngroup by 1\norder by 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "seq_scan" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1m" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Seq. scans ($agg_interval rate, >10MB tables)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "", + "fill": 1, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 11, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_lockmode", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "lockmode" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "locks_mode", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n lockmode,\n max(lock_count) as lock_count\nfrom (\n select \n (data->>'count')::int8 as lock_count,\n tag_data->>'lockmode' as lockmode,\n time\n from locks_mode\n where dbname = '$dbname' and $__timeFilter(time)\n and tag_data->>'lockmode' LIKE '%Exclusive%'\n) x\ngroup by 1, 2\norder by 1, 2\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "count" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + }, + { + "condition": "AND", + "key": "lockmode", + "operator": "=~", + "value": "/Exclusive/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Exclusive locks ($agg_interval max.)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Brought to you by: \"Cybertec", + "editable": true, + "error": false, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 14, + "links": [], + "mode": "html", + "options": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 19, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "", + "value": "" + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "tags": [], + "text": "5m", + "value": "5m" + }, + "hide": 0, + "label": null, + "name": "agg_interval", + "options": [ + { + "selected": false, + "text": "1s", + "value": "1s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1s,1m,5m,10m,15m,30m,1h,6h,12h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "DB overview Unprivileged / Developer mode", + "uid": "db-overview-developer", + "version": 1 + } + db-overview-time-lag.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "Active + idle in transaction user sessions", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 12, + "interval": "", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n coalesce(avg((data->>'active')::int8 + (data->>'idleintransaction')::int8), 0) as active\nfrom backends\n where dbname = '$dbname'\n and $__timeFilter(time)\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n coalesce(avg(active), 0) as \"active -$lag_interval\"\nfrom (\nselect\n time + '$lag_interval'::interval as time,\n (data->>'active')::int8 + (data->>'idleintransaction')::int8 as active\nfrom backends\n where dbname = '$dbname'\n and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n) x\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active sessions", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 2, + "interval": "", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((c - c_lag + r - r_lag) / extract(epoch from time - time_lag)) as \"TPS\"\nfrom (\n select \n (data->>'xact_commit')::int8 as c, lag((data->>'xact_commit')::int8) over w as c_lag,\n (data->>'xact_rollback')::int8 as r, lag((data->>'xact_rollback')::int8) over w as r_lag,\n time, lag(time) over w as time_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and r >= r_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((c - c_lag + r - r_lag) / extract(epoch from time - time_lag)) as \"TPS-$lag_interval\"\nfrom (\n select \n (data->>'xact_commit')::int8 as c, lag((data->>'xact_commit')::int8) over w as c_lag,\n (data->>'xact_rollback')::int8 as r, lag((data->>'xact_rollback')::int8) over w as r_lag,\n time + '$lag_interval'::interval as time, lag(time) over w + '$lag_interval'::interval as time_lag\n from db_stats\n where dbname = '$dbname' and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n window w as (order by time) \n) x\nwhere c >= c_lag and r >= r_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 3, + "interval": "", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((c - c_lag) / extract(epoch from time - time_lag)) as \"QPS\"\nfrom (\n select \n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from stat_statements_calls\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and time > time_lag\ngroup by 1\norder by 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((c - c_lag + r - r_lag) / extract(epoch from time - time_lag)) as \"QPS-$lag_interval\"\nfrom (\n select \n (data->>'xact_commit')::int8 as c, lag((data->>'xact_commit')::int8) over w as c_lag,\n (data->>'xact_rollback')::int8 as r, lag((data->>'xact_rollback')::int8) over w as r_lag,\n time + '$lag_interval'::interval as time, lag(time) over w + '$lag_interval'::interval as time_lag\n from db_stats\n where dbname = '$dbname' and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n window w as (order by time) \n) x\nwhere c >= c_lag and r >= r_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "QPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Based on pg_stat_statements data", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 4, + "interval": "", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n time,\n avg(avg) as \"avg_query_runtime\"\nfrom (\n\nselect\n$__timeGroup(time, $agg_interval),\nqueryid,\navg((tt-tt_lag)::numeric / (c-c_lag))\n\nfrom (\n\n select \n (data->>'total_time')::float8 as tt, lag((data->>'total_time')::float8) over w as tt_lag,\n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n tag_data->>'queryid' as queryid,\n time\n from stat_statements\n where dbname = '$dbname' and $__timeFilter(time)\n and not tag_data->>'query' like '%$2)::int8 as epoch_ns,%' /*trying to not include queries by pgwatch itself*/\n window w as (partition by tag_data->>'queryid' order by time)\n) x\nwhere c > c_lag and tt >= tt_lag\ngroup by 1, 2\norder by 1, 2\n) y\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n time,\n avg(avg) as \"avg_query_runtime-$lag_interval\"\nfrom (\n\nselect\n$__timeGroup(time, $agg_interval),\nqueryid,\navg((tt-tt_lag)::numeric / (c-c_lag))\n\nfrom (\n\n select \n (data->>'total_time')::float8 as tt, lag((data->>'total_time')::float8) over w as tt_lag,\n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n tag_data->>'queryid' as queryid,\n time + '$lag_interval'::interval as time\n from stat_statements\n where dbname = '$dbname' and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n and not tag_data->>'query' like '%$2)::int8 as epoch_ns,%' /*trying to not include queries by pgwatch itself*/\n window w as (partition by tag_data->>'queryid' order by time)\n) x\nwhere c > c_lag and tt >= tt_lag\ngroup by 1, 2\norder by 1, 2\n) y\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Avg. query runtime", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 5, + "interval": "", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((wal-wal_lag) / (etime-lag_etime)) as \"WAL rate\"\nfrom (\n select \n (data->>'xlog_location_b')::int8 as wal, lag((data->>'xlog_location_b')::int8) over w as wal_lag,\n extract(epoch from time) as etime,\n lag(extract(epoch from time)) over w as lag_etime,\n time\n from wal\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere wal >= wal_lag and etime - lag_etime > 0\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((wal-wal_lag) / (etime-lag_etime)) as \"WAL rate-$lag_interval\"\nfrom (\n select \n (data->>'xlog_location_b')::int8 as wal, lag((data->>'xlog_location_b')::int8) over w as wal_lag,\n extract(epoch from time + '$lag_interval'::interval) as etime,\n lag(extract(epoch from time + '$lag_interval'::interval)) over w as lag_etime,\n time + '$lag_interval'::interval as time\n from wal\n where dbname = '$dbname' and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n window w as (order by time)\n) x\nwhere wal >= wal_lag and etime - lag_etime > 0\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "WAL rate", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Temp tables not included", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 40 + }, + "hiddenSeries": false, + "id": 6, + "interval": "", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(((ins-ins_lag) * (extract(epoch from '$agg_interval'::interval))) / extract(epoch from time - time_lag)) as \"INSERT\"\nfrom (\n select \n (data->>'tup_inserted')::int8 as ins, lag((data->>'tup_inserted')::int8) over w as ins_lag,\n time, lag(time) over w as time_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere ins >= ins_lag\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(((ins-ins_lag) * (extract(epoch from '$agg_interval'::interval))) / extract(epoch from time - time_lag)) as \"INSERT-$lag_interval\"\nfrom (\n select \n (data->>'tup_inserted')::int8 as ins, lag((data->>'tup_inserted')::int8) over w as ins_lag,\n time + '$lag_interval'::interval as time, lag(time) over w + '$lag_interval'::interval as time_lag\n from db_stats\n where dbname = '$dbname' and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n window w as (order by time)\n) x\nwhere ins >= ins_lag\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "INSERT-s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Temp tables not included", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "hiddenSeries": false, + "id": 7, + "interval": "", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(((upd-upd_lag) * (extract(epoch from '$agg_interval'::interval))) / extract(epoch from time - time_lag)) as \"UPDATE\"\nfrom (\n select \n (data->>'tup_updated')::int8 as upd, lag((data->>'tup_updated')::int8) over w as upd_lag,\n time, lag(time) over w as time_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere upd >= upd_lag\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(((upd-upd_lag) * (extract(epoch from '$agg_interval'::interval))) / extract(epoch from time - time_lag)) as \"UPDATE-$lag_interval\"\nfrom (\n select \n (data->>'tup_updated')::int8 as upd, lag((data->>'tup_updated')::int8) over w as upd_lag,\n time + '$lag_interval'::interval as time, lag(time) over w + '$lag_interval'::interval as time_lag\n from db_stats\n where dbname = '$dbname' and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n window w as (order by time)\n) x\nwhere upd >= upd_lag\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "UPDATE-s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Temp tables not included", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "hiddenSeries": false, + "id": 8, + "interval": "", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(((del-del_lag) * (extract(epoch from '$agg_interval'::interval))) / extract(epoch from time - time_lag)) as \"delATE\"\nfrom (\n select \n (data->>'tup_deleted')::int8 as del, lag((data->>'tup_deleted')::int8) over w as del_lag,\n time, lag(time) over w as time_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere del >= del_lag\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(((del-del_lag) * (extract(epoch from '$agg_interval'::interval))) / extract(epoch from time - time_lag)) as \"DELETE-$lag_interval\"\nfrom (\n select \n (data->>'tup_deleted')::int8 as del, lag((data->>'tup_deleted')::int8) over w as del_lag,\n time + '$lag_interval'::interval as time, lag(time) over w + '$lag_interval'::interval as time_lag\n from db_stats\n where dbname = '$dbname' and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n window w as (order by time)\n) x\nwhere del >= del_lag\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DELETE-s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "hiddenSeries": false, + "id": 9, + "interval": "", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n time,\n sum(seq_scan) as seq_scan\nfrom (\n select\n $__timeGroup(time, $agg_interval),\n table_full_name,\n avg((((scan - scan_lag) * (extract(epoch from '$agg_interval'::interval))) / extract(epoch from time - time_lag))) as seq_scan\n from (\n select \n (data->>'seq_scan')::int8 as scan, lag((data->>'seq_scan')::int8) over w as scan_lag,\n time, lag(time) over w as time_lag,\n tag_data->>'table_full_name' as table_full_name\n from table_stats\n where dbname = '$dbname' and $__timeFilter(time)\n and (data->>'table_size_b')::int8 > 1e8\n window w as (partition by tag_data->>'table_full_name' order by time) \n ) x\n where scan >= scan_lag and time > time_lag\n group by 1, 2\n) y\ngroup by 1\norder by 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n time,\n sum(seq_scan) as \"seq_scan-$lag_interval\"\nfrom (\n select\n $__timeGroup(time, $agg_interval),\n table_full_name,\n avg((((scan - scan_lag) * (extract(epoch from '$agg_interval'::interval))) / extract(epoch from time - time_lag))) as seq_scan\n from (\n select \n (data->>'seq_scan')::int8 as scan, lag((data->>'seq_scan')::int8) over w as scan_lag,\n time + '$lag_interval'::interval as time, lag(time) over w + '$lag_interval'::interval as time_lag,\n tag_data->>'table_full_name' as table_full_name\n from table_stats\n where dbname = '$dbname' and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n and (data->>'table_size_b')::int8 > 1e8\n window w as (partition by tag_data->>'table_full_name' order by time) \n ) x\n where scan >= scan_lag and time > time_lag\n group by 1, 2\n) y\ngroup by 1\norder by 1\n", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Seq. scans on tables > 100MB", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 72 + }, + "hiddenSeries": false, + "id": 10, + "interval": "", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n sum(tb-tb_lag) as \"Temp bytes\"\nfrom (\n select \n time,\n lag(time) over w as time_lag, \n (data->>'temp_bytes')::int8 as tb,\n lag((data->>'temp_bytes')::int8) over w as tb_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere tb >= tb_lag\ngroup by 1\norder by 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n sum(tb-tb_lag) as \"Temp bytes -$lag_interval\"\nfrom (\n select \n time + '$lag_interval'::interval as time,\n (data->>'temp_bytes')::int8 as tb,\n lag((data->>'temp_bytes')::int8) over w as tb_lag\n from db_stats\n where dbname = '$dbname' and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n window w as (order by time)\n) x\nwhere tb >= tb_lag\ngroup by 1\norder by 1\n", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Temp bytes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "i.e. the Postgres cache ratio", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 80 + }, + "hiddenSeries": false, + "id": 11, + "interval": "", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((hit-hit_lag)::numeric / ((hit-hit_lag) + (read-read_lag)))*100 as \"SB hit ratio\"\nfrom (\n select\n time,\n (data->>'blks_hit')::int8 as hit, lag((data->>'blks_hit')::int8) over w as hit_lag,\n (data->>'blks_read')::int8 as read, lag((data->>'blks_read')::int8) over w as read_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere hit > hit_lag or read > read_lag\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((hit-hit_lag)::numeric / ((hit-hit_lag) + (read-read_lag)))*100 as \"SB hit ratio -$lag_interval\"\nfrom (\n select\n time + '$lag_interval'::interval as time,\n (data->>'blks_hit')::int8 as hit, lag((data->>'blks_hit')::int8) over w as hit_lag,\n (data->>'blks_read')::int8 as read, lag((data->>'blks_read')::int8) over w as read_lag\n from db_stats\n where dbname = '$dbname' and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n window w as (order by time)\n) x\nwhere hit > hit_lag or read > read_lag\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Shared buffers hit ratio", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Brought to you by: \"Cybertec", + "datasource": null, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 88 + }, + "id": 14, + "mode": "html", + "timeFrom": null, + "timeShift": null, + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 22, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": "test", + "value": "test" + }, + "datasource": null, + "definition": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics ORDER BY 1;", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "10m", + "value": "10m" + }, + "hide": 0, + "label": null, + "name": "agg_interval", + "options": [ + { + "selected": false, + "text": "1s", + "value": "1s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1s,1m,5m,10m,15m,30m,1h,6h,12h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "1d", + "value": "1d" + }, + "hide": 0, + "label": null, + "name": "lag_interval", + "options": [ + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": true, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "DB overview time lag comparison", + "uid": "db-overview-time-lag", + "variables": { + "list": [] + }, + "version": 1 + } + db-overview.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "description": "Last 5min data", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 0 + }, + "hideTimeOverride": true, + "id": 16, + "interval": "10m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "alias": "TPS", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time,\n (data->'in_recovery_int')::int\nFROM\n wal\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nORDER BY time DESC\nLIMIT 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "5m", + "title": "Instance state", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "PRIMARY", + "value": "0" + }, + { + "op": "=", + "text": "REPLICA", + "value": "1" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "description": "Last 5min data", + "editable": true, + "error": false, + "format": "dtdurations", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 3, + "y": 0 + }, + "hideTimeOverride": true, + "id": 15, + "interval": "", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "alias": "TPS", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time,\n (data->'postmaster_uptime_s')::int\nFROM\n wal\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nORDER BY time DESC\nLIMIT 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "5m", + "title": "Instance uptime", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "description": "Current Transaction Per Second (10min avg.) + 6h history", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 6, + "y": 0 + }, + "hideTimeOverride": true, + "id": 1, + "interval": "10m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "TPS", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $__interval),\n avg(((c - c_lag) + (r - r_lag)) / extract(epoch from time - time_lag)) as tps\nfrom (\n select \n (data->>'xact_commit')::int8 as c, lag((data->>'xact_commit')::int8) over w as c_lag,\n (data->>'xact_rollback')::int8 as r, lag((data->>'xact_rollback')::int8) over w as r_lag,\n time, lag(time) over w as time_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and r >= r_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "6h", + "title": "TPS", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "description": "Queries Per Second based on pg_stat_statements.calls. 10min avg. + 6h history", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 9, + "y": 0 + }, + "height": "150", + "hideTimeOverride": true, + "id": 2, + "interval": "10m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "QPS", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $__interval),\n avg((c - c_lag) / extract(epoch from time - time_lag)) as qps\nfrom (\n select \n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from stat_statements_calls\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag\ngroup by 1\norder by 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "6h", + "title": "QPS", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "description": "10min avg. + 6h history. Requires pg_stat_statements extension and the \"get_stat_statements\" helper function from the \"sql/metric_fetching_helpers\" folder on the monitored DB. Not including pgwatch generated metrics fetching queries.", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 12, + "y": 0 + }, + "height": "150", + "hideTimeOverride": true, + "id": 3, + "interval": "10m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ms", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "avg_query_runtime_per_db", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n time,\n avg(avg) as \"avg_query_runtime\"\nfrom (\n\nselect\n$__timeGroup(time, $__interval),\nqueryid,\navg((tt-tt_lag)::numeric / (c-c_lag))\n\nfrom (\n\n select \n (data->>'total_time')::float8 as tt, lag((data->>'total_time')::float8) over w as tt_lag,\n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n tag_data->>'queryid' as queryid,\n time\n from stat_statements\n where dbname = '$dbname' and $__timeFilter(time)\n and not tag_data->>'query' like '%$2)::int8 as epoch_ns,%' /*trying to not include queries by pgwatch itself*/\n window w as (partition by tag_data->>'queryid' order by time)\n) x\nwhere c > c_lag and tt >= tt_lag\ngroup by 1, 2\norder by 1, 2\n) y\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "6h", + "timeShift": null, + "title": "Query runtime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": 1, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 15, + "y": 0 + }, + "height": "150", + "hideTimeOverride": true, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "db_size_change_last_hour", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "select\n time,\n last_value - first_value\nfrom (\nselect\n time,\n max(first_value) as first_value,\n max(last_value) as last_value\nfrom (\n select\n date_trunc('hour', time) as time,\n first_value((data->>'size_b')::int8) over w,\n last_value((data->>'size_b')::int8) over w\n from\n db_size\n where \n dbname = '$dbname' and $__timeFilter(time)\n window w as (\n partition by date_trunc('hour', time) order by time rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\n )\n) x\ngroup by 1\nhaving count(*) > 1 -- if only 1 entry in time block, diff will be always 0\n) y\norder by 1;", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "6h", + "timeShift": null, + "title": "DB size ch. 1h", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": 1, + "description": "Based on a pure SQL based estimate for fast results. Re-check with \"pgstattuple\" if numbers look suspicious!", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 18, + "y": 0 + }, + "height": "150", + "hideTimeOverride": true, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "Bloat size", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1h" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_bloat_approx_summary", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "SELECT\n time,\n avg((data->'approx_table_bloat_b')::int8)\nFROM\n table_bloat_approx_summary_sql\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY\n 1\nORDER BY\n 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "approx_free_space_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "6h", + "timeShift": null, + "title": "Approx Table Bloat", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": 1, + "description": "Percentage of rows touched by queries that are actually returned to the client. Low numbers (< 10%) for an OLTP workload can hint at missing indexes", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 21, + "y": 0 + }, + "height": "150", + "hideTimeOverride": true, + "id": 6, + "interval": "10m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "load_5", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "cpu_load", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $__interval),\n 100 * avg((fetched - fetched_lag)::numeric / (returned - returned_lag)) as fetched_vs_returned_pct\nFROM (\n\nSELECT\n time,\n (data->'tup_fetched')::int8 as fetched,\n lag((data->'tup_fetched')::int8) over w as fetched_lag,\n (data->'tup_returned')::int8 as returned,\n lag((data->'tup_returned')::int8) over w as returned_lag\nFROM\n db_stats\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nWINDOW w AS (ORDER BY time)\n) x \nWHERE returned > returned_lag AND fetched >= fetched_lag\nGROUP BY 1\nORDER BY 1\n\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load_5min" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "6h", + "timeShift": null, + "title": "Tuples fetched vs returned", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "", + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 2 + }, + "hiddenSeries": false, + "id": 7, + "interval": "5m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "db_stats.mean", + "yaxis": 2 + }, + { + "alias": "DB size", + "yaxis": 2 + }, + { + "alias": "blk_read_time", + "yaxis": 2 + }, + { + "alias": "blk_write_time", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "DELETE", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(((ins-ins_lag) * (extract(epoch from '$agg_interval'::interval))) / extract(epoch from time - time_lag)) as \"INSERT\",\n avg(((upd-upd_lag) * (extract(epoch from '$agg_interval'::interval))) / extract(epoch from time - time_lag)) as \"UPDATE\",\n avg(((del-del_lag) * (extract(epoch from '$agg_interval'::interval))) / extract(epoch from time - time_lag)) as \"DELETE\"\nfrom (\n select \n (data->>'tup_inserted')::int8 as ins, lag((data->>'tup_inserted')::int8) over w as ins_lag,\n (data->>'tup_updated')::int8 as upd, lag((data->>'tup_updated')::int8) over w as upd_lag,\n (data->>'tup_deleted')::int8 as del, lag((data->>'tup_deleted')::int8) over w as del_lag,\n time, lag(time) over w as time_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere ins >= ins_lag and upd >= upd_lag and del >= del_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_deleted" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Tuple ins. / upd. / del. statistics ($agg_interval avg. rate, log. scale)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "TX rollback ratio": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 2 + }, + "hiddenSeries": false, + "id": 8, + "interval": "5m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Shared buffers hit ratio", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((hit-hit_lag)::numeric / ((hit-hit_lag) + (read-read_lag)))*100 as \"Shared buffers hit ratio\",\n avg((roll-roll_lag)::numeric / ((roll-roll_lag) + (comm-comm_lag)))*100 as \"TX rollback ratio\"\nfrom (\n select \n (data->>'blks_hit')::int8 as hit, lag((data->>'blks_hit')::int8) over w as hit_lag,\n (data->>'blks_read')::int8 as read, lag((data->>'blks_read')::int8) over w as read_lag,\n (data->>'xact_rollback')::int8 as roll, lag((data->>'xact_rollback')::int8) over w as roll_lag,\n (data->>'xact_commit')::int8 as comm, lag((data->>'xact_commit')::int8) over w as comm_lag,\n time\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere (hit > hit_lag or read > read_lag) and (roll > roll_lag or comm > comm_lag)\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "blks_hit" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Shared Buffers hit ratio + Rollback ratio ($agg_interval avg.)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": "105", + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "# Sessions": "#cca300", + "#Backends": "#F9E2D2", + "Deadlocks": "dark-red", + "Deadlocks (1h rate)": "#BF1B00", + "QPS": "dark-green", + "TPS": "dark-orange", + "Temp bytes (1h rate)": "#6ed0e0" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 9, + "interval": "", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "# Sessions", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((c - c_lag + r - r_lag) / extract(epoch from time - time_lag)) as \"TPS\"\nfrom (\n select \n (data->>'xact_commit')::int8 as c, lag((data->>'xact_commit')::int8) over w as c_lag,\n (data->>'xact_rollback')::int8 as r, lag((data->>'xact_rollback')::int8) over w as r_lag,\n time, lag(time) over w as time_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere time > time_lag and c >= c_lag\ngroup by 1\norder by 1", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "numbackends" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((c - c_lag) / extract(epoch from time - time_lag)) as \"QPS\"\nfrom (\n select \n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from stat_statements_calls\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere time > time_lag and c >= c_lag\ngroup by 1\norder by 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TPS / QPS ($agg_interval avg.)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "DB Size": "#2F575E", + "WAL rate": "#0A50A1" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 8 + }, + "height": "", + "hiddenSeries": false, + "id": 10, + "interval": "2m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "DB Size", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "WAL rate", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "wal", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((wal-wal_lag) / (etime-lag_etime)) as \"WAL rate\"\nfrom (\n select \n (data->>'xlog_location_b')::int8 as wal, lag((data->>'xlog_location_b')::int8) over w as wal_lag,\n extract(epoch from time) as etime,\n lag(extract(epoch from time)) over w as lag_etime,\n time\n from wal\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere wal >= wal_lag and etime - lag_etime > 0\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "xlog_location_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1h" + ], + "type": "derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(size_b) as \"DB Size\"\nfrom (\n select \n (data->>'size_b')::int8 as size_b,\n time\n from db_size\n where dbname = '$dbname' and $__timeFilter(time)\n) x\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "WAL rate + DB size ($agg_interval avg.)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "idle in transaction": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "Based on get_stat_activity() helper", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 11, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "idle", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "backends", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((data->>'idle')::int8) as idle,\n avg((data->>'idleintransaction')::int8) as \"idle in transaction\",\n avg((data->>'waiting')::int8) as waiting,\n avg((data->>'active')::int8) as active,\n avg((data->>'av_workers')::int8) as av_workers\nfrom backends\n where dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "idle" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sessions by state ($agg_interval avg.)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "load_5": "#BA43A9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "Requires according \"metric fetching helpers\" on the monitored server. Not including pgwatch generated metrics fetching queries.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 12, + "interval": "10m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "avg_query_runtime", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "load_5", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "cpu_load", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((data->>'load_5min')::numeric) as \"load_5\"\nFROM\n cpu_load\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1\nORDER BY 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load_5min" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "alias": "avg_query_runtime", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "stat_statements", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n time,\n avg(avg) as \"avg_query_runtime\"\nfrom (\n\nselect\n$__timeGroup(time, $agg_interval),\nqueryid,\navg((tt-tt_lag)::numeric / (c-c_lag))\n\nfrom (\n\n select \n (data->>'total_time')::float8 as tt, lag((data->>'total_time')::float8) over w as tt_lag,\n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n tag_data->>'queryid' as queryid,\n time\n from stat_statements\n where dbname = '$dbname' and $__timeFilter(time)\n and not tag_data->>'query' like '%$2)::int8 as epoch_ns,%' /*trying to not include queries by pgwatch itself*/\n window w as (partition by tag_data->>'queryid' order by time)\n) x\nwhere c > c_lag and tt >= tt_lag\ngroup by 1, 2\norder by 1, 2\n) y\ngroup by 1\norder by 1", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU load + avg. query runtime ($agg_interval avg.)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 1, + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n time,\n sum(seq_scan) as seq_scan,\n sum(idx_scan) as idx_scan\nfrom (\n select\n $__timeGroup(time, $agg_interval),\n table_full_name,\n avg((((scan - scan_lag) * (extract(epoch from '$agg_interval'::interval))) / extract(epoch from time - time_lag))) as seq_scan,\n avg((((idx_scan - idx_scan_lag) * (extract(epoch from '$agg_interval'::interval))) / extract(epoch from time - time_lag))) as idx_scan\n from (\n select \n (data->>'seq_scan')::int8 as scan, lag((data->>'seq_scan')::int8) over w as scan_lag,\n (data->>'idx_scan')::int8 as idx_scan, lag((data->>'idx_scan')::int8) over w as idx_scan_lag,\n time, lag(time) over w as time_lag,\n tag_data->>'table_full_name' as table_full_name\n from table_stats\n where dbname = '$dbname' and $__timeFilter(time)\n and (data->>'table_size_b')::int8 > 10000000\n window w as (partition by tag_data->>'table_full_name' order by time) \n ) x\n where scan >= scan_lag and time > time_lag\n group by 1, 2\n) y\ngroup by 1\norder by 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Seq. / Idx. scans ($agg_interval avg. rate., >10MB tables, log. scale)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Temp bytes": "light-red", + "Temp bytes (1h rate)": "light-red", + "Temp bytes (5m avg)": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 0, + "description": "Appears when larger groupings / sortings don't fit into 'work_mem'", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 20 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(((tb-tb_lag)*extract(epoch from '$agg_interval'::interval)) / (etime - lag_etime)) as \"Temp bytes\"\nfrom (\n select \n (data->>'temp_bytes')::int8 as tb, lag((data->>'temp_bytes')::int8) over w as tb_lag,\n (extract(epoch from time))::int8 as etime,\n (lag(extract(epoch from time)) over w)::int8 as lag_etime,\n time\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere tb >= tb_lag\ngroup by 1\norder by 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Temp bytes ($agg_interval avg.)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "decimals": 1, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Brought to you by: \"Cybertec", + "datasource": null, + "editable": true, + "error": false, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 14, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": false, + "schemaVersion": 22, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "5m", + "value": "5m" + }, + "hide": 0, + "label": null, + "name": "agg_interval", + "options": [ + { + "selected": false, + "text": "1s", + "value": "1s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1s,1m,5m,10m,15m,30m,1h,6h,12h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "DB overview", + "uid": "db-overview", + "version": 1 + } + global-health.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": null, + "description": "Please adjust the markers as needed! Also the default query assumes that all DBs are separate instances. If not so change the query to add some filters or groupings based on the wal.sys_id column for example or the configured_dbs.continuous_discovery_prefix if using the DB auto-discovery modes. The same applies also for the below \"top\" table panels.", + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 10, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-green", + "value": 10 + } + ] + } + }, + "overrides": [], + "values": false + }, + "orientation": "auto", + "showThresholdLabels": true, + "showThresholdMarkers": true + }, + "pluginVersion": "6.6.1", + "targets": [ + { + "format": "time_series", + "group": [ + { + "params": [ + "dbname" + ], + "type": "column" + } + ], + "metricColumn": "dbname", + "rawQuery": true, + "rawSql": "SELECT\n 0 AS \"time\",\n count(distinct dbname)\nFROM wal\nWHERE\n $__timeFilter(\"time\")\n AND (data->'in_recovery_int')::int = 0", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + }, + { + "params": [ + "avg" + ], + "type": "aggregate" + }, + { + "params": [ + "value" + ], + "type": "alias" + } + ] + ], + "table": "db_stats", + "timeColumn": "\"time\"", + "timeColumnType": "timestamptz", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Monitored primary DB-s ", + "type": "gauge" + }, + { + "datasource": null, + "description": "Please adjust the markers as needed! Also the default query assumes that all DBs are separate instances. If not so change the query to add some filters or groupings based on the wal.sys_id column for example or the configured_dbs.continuous_discovery_prefix if using the DB auto-discovery modes. The same applies also for the below \"top\" table panels.", + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 3, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 10, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-orange", + "value": 9 + }, + { + "color": "dark-green", + "value": 10 + } + ] + } + }, + "overrides": [], + "values": false + }, + "orientation": "auto", + "showThresholdLabels": true, + "showThresholdMarkers": true + }, + "pluginVersion": "6.6.1", + "targets": [ + { + "format": "time_series", + "group": [ + { + "params": [ + "dbname" + ], + "type": "column" + } + ], + "metricColumn": "dbname", + "rawQuery": true, + "rawSql": "SELECT\n 0 AS \"time\",\n count(distinct dbname)\nFROM\n wal\nWHERE\n $__timeFilter(\"time\")\n AND (data->'in_recovery_int')::int = 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + }, + { + "params": [ + "avg" + ], + "type": "aggregate" + }, + { + "params": [ + "value" + ], + "type": "alias" + } + ] + ], + "table": "db_stats", + "timeColumn": "\"time\"", + "timeColumnType": "timestamptz", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Monitored replica DB-s", + "type": "gauge" + }, + { + "columns": [], + "datasource": null, + "description": "Configured to be monitored but no recent data", + "fontSize": "110%", + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 5, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "", + "align": "auto", + "colorMode": "row", + "colors": [ + "#FA6400", + "#E02F44", + "#C4162A" + ], + "decimals": null, + "pattern": "last_seen", + "thresholds": [ + "600", + "3600" + ], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "last_seen_uptime", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Go to Overview dash", + "linkUrl": "/d/db-overview?var-dbname=${__cell}", + "mappingType": 1, + "pattern": "dbname", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n (SELECT case when data->>'in_recovery_int' = '0' then 'primary' else 'replica' end from wal where time > now() - '15min'::interval and dbname = c.dbname order by time desc limit 1) as last_role,\n (SELECT (extract(epoch from now() - time))::int from db_stats where dbname = c.dbname order by time desc limit 1) as last_seen,\n (SELECT (data->'postmaster_uptime_s')::int from db_stats where dbname = c.dbname order by time desc limit 1) as last_seen_uptime\nFROM\n (select dbname, max(time) from configured_dbs where time between now() - '15min'::interval and now() group by dbname) c /* 10min ping interval hardcoded in collector */\nWHERE\n NOT EXISTS (\n select dbname from db_stats where $__timeFilter(\"time\") and dbname = c.dbname union \n select dbname from pgbouncer_stats where $__timeFilter(\"time\") and dbname = c.dbname \n)", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Offline nodes", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Last $__interval average. 2 /5 % threshold defaults.", + "fontSize": "90%", + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 7, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "", + "align": "auto", + "colorMode": "value", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "decimals": 1, + "pattern": "rollback_ratio", + "thresholds": [ + "2", + "5" + ], + "type": "number", + "unit": "none" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Go to 'Overview' dash", + "linkUrl": "/d/db-overview?var-dbname=${__cell}", + "mappingType": 1, + "pattern": "dbname", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select * from (\nselect\n dbname,\n avg((roll-roll_lag)::numeric / ((roll-roll_lag) + (comm-comm_lag)))*100 as \"rollback_ratio\"\nfrom (\n select\n dbname,\n time,\n (data->>'xact_rollback')::int8 as roll, lag((data->>'xact_rollback')::int8) over w as roll_lag,\n (data->>'xact_commit')::int8 as comm, lag((data->>'xact_commit')::int8) over w as comm_lag\n FROM\n db_stats\n WHERE\n $__timeFilter(\"time\")\n window w as (partition by dbname order by time)\n) x\nWHERE comm > comm_lag or roll > roll_lag\nGROUP BY 1\n) y\nWHERE rollback_ratio > 0\nORDER BY 2 DESC\nLIMIT $top_limit\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $top_limit by TX rollback %", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Includes \"idle in transaction\" TX. 5 / 10 min default thresholds", + "fontSize": "90%", + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 7 + }, + "id": 10, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "", + "align": "auto", + "colorMode": "value", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "longest_tx_seconds", + "thresholds": [ + "300", + "600" + ], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Go to 'Overview' dash", + "linkUrl": "/d/db-overview?var-dbname=${__cell}", + "mappingType": 1, + "pattern": "dbname", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT * FROM (\nSELECT\n dbname,\n max((data->'longest_tx_seconds')::int) AS longest_tx_seconds\nFROM\n backends\nWHERE\n $__timeFilter(\"time\")\nGROUP BY\n dbname\n) x\nWHERE longest_tx_seconds > 0\nORDER BY 2 DESC\nLIMIT $top_limit", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $top_limit by longest TX time", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Assumes get_stat_activity() helper for non-privileged monitoring users", + "fontSize": "90%", + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 7 + }, + "id": 6, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Go to Overview dash", + "linkUrl": "/d/db-overview?var-dbname=${__cell}", + "mappingType": 1, + "pattern": "dbname", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": "value", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "waiting", + "thresholds": [ + "1", + "5" + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n max((data->'waiting')::int) AS \"waiting\"\nFROM\n backends\nWHERE\n $__timeFilter(\"time\")\n AND (data->'waiting')::int > 0\nGROUP BY\n 1\nORDER BY\n 2 DESC NULLS LAST\nLIMIT\n $top_limit", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $top_limit by blocked sessions", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Assumes get_stat_activity() helper for non-privileged monitoring users", + "fontSize": "90%", + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 14, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Go to Overview dash", + "linkUrl": "/d/db-overview?var-dbname=${__cell}", + "mappingType": 1, + "pattern": "dbname", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": "value", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "waiting", + "thresholds": [ + "1", + "5" + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n round(100 * instance_total::numeric / max_connections, 1) AS \"pct_used\",\n instance_total AS used_connections,\n max_connections\nFROM\n (\n select\n distinct on (dbname)\n dbname,\n (data->'max_connections')::int as max_connections,\n (data->'instance_total')::int as instance_total\n FROM\n backends\n WHERE\n $__timeFilter(time)\n order by dbname, time desc\n ) x\nORDER BY\n 2 DESC NULLS LAST\nLIMIT\n $top_limit", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $top_limit by used connections", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "100 MB / 1 GB default threshold", + "fontSize": "90%", + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 12 + }, + "id": 9, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "", + "align": "auto", + "colorMode": "value", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "temp_bytes_per_minute", + "thresholds": [ + "100000000", + "1000000000" + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Go to 'Overview' dash", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Go to 'Overview' dash", + "linkUrl": "/d/db-overview?var-dbname=${__cell}", + "mappingType": 1, + "pattern": "dbname", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select * from (\nselect\n dbname,\n avg((tb-tb_lag) / (etime - lag_etime))*60 as \"temp_bytes_per_minute\"\nfrom (\n select\n dbname,\n (data->>'temp_bytes')::int8 as tb, lag((data->>'temp_bytes')::int8) over w as tb_lag,\n (extract(epoch from time))::int8 as etime,\n (lag(extract(epoch from time)) over w)::int8 as lag_etime\n from db_stats\n where $__timeFilter(time)\n window w as (partition by dbname order by time)\n) x\nwhere\n tb >= tb_lag and etime > lag_etime\ngroup by\n dbname\n) y\nwhere temp_bytes_per_minute > 0\norder by 2 desc\nlimit $top_limit", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $top_limit by temp. files", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "1 / 16 MB default thresholds", + "fontSize": "90%", + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 8, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "", + "align": "auto", + "colorMode": "value", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "replay_lag_b", + "thresholds": [ + "1000000", + "16000000" + ], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Go to 'Replication' dash", + "linkUrl": "/d/replication?var-dbname=${__cell}", + "mappingType": 1, + "pattern": "dbname", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select * from (\nselect \n dbname,\n max((data->'replay_lag_b')::int8) as replay_lag_b\nfrom\n replication\nwhere\n (data->'replay_lag_b')::int8 > 0\ngroup by\n dbname\n) x\norder by 2 desc\nlimit $top_limit", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $top_limit by replication lag", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Assumes \"psutil_disk\" helper. 10 / 5% default warning thresholds", + "fontSize": "90%", + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 11, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "", + "align": "auto", + "colorMode": "value", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "used_percent", + "thresholds": [ + "80", + "90" + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Go to 'System Stats' dash", + "linkUrl": "/d/system-stats?var-dbname=${__cell}", + "mappingType": 1, + "pattern": "dbname", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select * from (\nselect \n dbname,\n max((data->'percent')::float) as used_percent\nfrom\n psutil_disk\nwhere\n $__timeFilter(\"time\")\n and (data->'percent')::float > 0\ngroup by\n dbname\n) x\norder by 2 desc nulls last\nlimit $top_limit", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Lowest $top_limit by free disk %", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Assumes \"psutil_cpu\" helpers are installed and configured", + "fontSize": "90%", + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 15, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Go to Overview dash", + "linkUrl": "/d/db-overview?var-dbname=${__cell}", + "mappingType": 1, + "pattern": "dbname", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": "value", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "cpu_utilization", + "thresholds": [ + "30", + "60" + ], + "type": "number", + "unit": "percent" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n round(max((data->'cpu_utilization')::numeric), 1) AS \"cpu_utilization\"\nFROM\n psutil_cpu\nWHERE\n $__timeFilter(\"time\")\nGROUP BY\n 1\nORDER BY\n 2 DESC NULLS LAST\nLIMIT\n $top_limit", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $top_limit by CPU utilization %", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Needs \"get_wal_size\" helper for unprivileged monitoring users", + "fontSize": "90%", + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 16, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Go to Overview dash", + "linkUrl": "/d/db-overview?var-dbname=${__cell}", + "mappingType": 1, + "pattern": "dbname", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "wal_size", + "thresholds": [], + "type": "number", + "unit": "bytes" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n max((data->'wal_size_b')::int8) as wal_size\nFROM\n wal_size\nWHERE\n $__timeFilter(\"time\")\nGROUP BY\n 1\nORDER BY\n 2 DESC NULLS LAST\nLIMIT\n $top_limit", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top $top_limit by WAL folder size", + "transform": "table", + "type": "table" + }, + { + "content": "Brought to you by: \"Cybertec", + "datasource": null, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 13, + "mode": "html", + "options": {}, + "timeFrom": null, + "timeShift": null, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "1m", + "schemaVersion": 22, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "3", + "value": "3" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "top_limit", + "options": [ + { + "selected": false, + "text": "1", + "value": "1" + }, + { + "selected": true, + "text": "3", + "value": "3" + }, + { + "selected": false, + "text": "5", + "value": "5" + }, + { + "selected": false, + "text": "10", + "value": "10" + } + ], + "query": "1,3,5,10", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Global Health", + "uid": "global-health", + "version": 1 + } + health-check.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Current and avg. KPI-s. Expects 'get_load_average' and 'backends' helpers to be installed on the monitored DB.", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "PRIMARY - accepting writes, REPLICA - read-only", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 0, + "y": 0 + }, + "hideTimeOverride": true, + "id": 28, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time,\n (data->'in_recovery_int')::int\nFROM\n wal\nWHERE\n time > now() - '$online_interval'::interval\n AND dbname = '$dbname'\nORDER BY time DESC\nLIMIT 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_relation_size_b" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1,2", + "timeFrom": "$online_interval", + "title": "Instance state", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "REPLICA", + "value": "1" + }, + { + "op": "=", + "text": "PRIMARY", + "value": "0" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": null, + "decimals": 0, + "description": "Time from last Postgres process restart. < 5m Error, < 30m Warn thresholds by default", + "format": "dtdurations", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 6, + "y": 0 + }, + "hideTimeOverride": true, + "id": 18, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time,\n (data->'postmaster_uptime_s')::int\nFROM\n wal\nWHERE\n time > now() - '$online_interval'::interval\n AND dbname = '$dbname'\nORDER BY time DESC\nLIMIT 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "postmaster_uptime_s" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "300, 1800", + "timeFrom": "$online_interval", + "title": "Instance uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#C4162A", + "#FA6400", + "#299c46" + ], + "datasource": null, + "decimals": 0, + "description": "\"SHOW server_version_num;\"\n110005 => 11.5", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 12, + "y": 0 + }, + "hideTimeOverride": true, + "id": 44, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "Postgres Version Overview", + "url": "/d/postgres-version-overview?$__all_variables" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication_slots", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select time, (data->'server_version_num')::int8 as value from settings where dbname = '$dbname' and $__timeFilter(time) order by time desc limit 1;", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "active" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "90400,90600", + "timeFrom": "6h", + "title": "PG version num.", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Longest query duration during last $online_interval", + "format": "dtdurations", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 18, + "y": 0 + }, + "hideTimeOverride": true, + "id": 32, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "backends", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n 0 AS time,\n max((data->'longest_query_seconds')::int)\nFROM\n backends\nWHERE\n time > now() - '$online_interval'::interval\n AND dbname = '$dbname'\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "longest_query_seconds" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "30,300", + "timeFrom": "$online_interval", + "title": "Longest query runtime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(237, 129, 40, 0.89)", + "#299c46", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Connected sessions, including Postgres internal processes like Autovacuum. < 1 Warn / > 300 Error thresholds by default", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 0, + "y": 2 + }, + "hideTimeOverride": true, + "id": 15, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'DB overview :: backends' panel", + "url": "/d/db-overview?$__all_variables&viewPanel=11&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time,\n (data->'numbackends')::int\nFROM\n db_stats\nWHERE\n time > now() - '$online_interval'::interval\n AND dbname = '$dbname'\nORDER BY time DESC\nLIMIT 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "numbackends" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1,300", + "timeFrom": "$online_interval", + "title": "Active connections", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "#FA6400", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "\"max_connections\" configuration setting. Changing requires re-start. 500/1000 Warn / Error thresholds by default", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 6, + "y": 2 + }, + "hideTimeOverride": true, + "id": 45, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'DB overview :: backends' panel", + "url": "/d/db-overview?$__all_variables&viewPanel=11&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time,\n (data->'max_connections')::int\nFROM\n settings\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nORDER BY time DESC\nLIMIT 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "numbackends" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "500,1000", + "timeFrom": "6h", + "title": "Max. connections", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Max number of queries waiting on resources locked by other sessions. 1 Warn / 5 Err thresholds by default", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 12, + "y": 2 + }, + "hideTimeOverride": true, + "id": 34, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'Lock details' dashboard", + "url": "/d/lock-details?$__all_variables" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "backends", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n max((data->'waiting')::int)\nFROM\n backends\nWHERE\n $__timeFilter(time, $online_interval)\n AND dbname = '$dbname'", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "waiting" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1,5", + "timeFrom": "$online_interval", + "title": "Blocked sessions", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": null, + "decimals": 1, + "description": "Shared Buffers is the Postgres -managed file cache and 90%+ is what you normally want to see for best performance.", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 18, + "y": 2 + }, + "hideTimeOverride": true, + "id": 21, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "Open 'DB overview :: share'd buffers ' panel", + "url": "/d/db-overview?$__all_variables&viewPanel=8&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1h" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n time,\n ((hit-hit_lag)::numeric / ((hit-hit_lag) + (read-read_lag)))*100 as \"Shared buffers hit ratio\"\nfrom (\n select \n (data->>'blks_hit')::int8 as hit, lag((data->>'blks_hit')::int8) over w as hit_lag,\n (data->>'blks_read')::int8 as read, lag((data->>'blks_read')::int8) over w as read_lag,\n (data->>'xact_rollback')::int8 as roll, lag((data->>'xact_rollback')::int8) over w as roll_lag,\n (data->>'xact_commit')::int8 as comm, lag((data->>'xact_commit')::int8) over w as comm_lag,\n time\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere hit > hit_lag or read > read_lag\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "blks_hit" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "50,80", + "timeFrom": null, + "title": "Shared Buffers hit pct.", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 1, + "description": "0.5% Warn / 1% Error thresholds by default", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 0, + "y": 4 + }, + "hideTimeOverride": true, + "id": 13, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'DB overview :: Rollback Ratio' panel", + "url": "/d/db-overview?$__all_variables&viewPanel=8&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $online_interval),\n max((roll-roll_lag)::numeric / ((roll-roll_lag) + (comm-comm_lag)))*100 as \"TX rollback ratio\"\nfrom (\n select\n time,\n (data->>'xact_rollback')::int8 as roll, lag((data->>'xact_rollback')::int8) over w as roll_lag,\n (data->>'xact_commit')::int8 as comm, lag((data->>'xact_commit')::int8) over w as comm_lag\n FROM\n db_stats\n WHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n window w as (order by time)\n) x\nWHERE comm > comm_lag or roll > roll_lag\nGROUP BY 1\nORDER BY 1 DESC\nLIMIT 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "xact_commit" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "0.5, 1", + "timeFrom": null, + "title": "TX rollback pct. (avg.)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": null, + "decimals": 1, + "description": "Transactions per Second. Last $online_interval average. <0.1 % Error / < 1.0 Warn thresholds by default", + "format": "short", + "gauge": { + "maxValue": null, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 6, + "y": 4 + }, + "hideTimeOverride": true, + "id": 7, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'DB overview Unprivileged / Developer :: TPS' panel", + "url": "/d/db-overview-developer?$__all_variables&viewPanel=15&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $online_interval),\n avg( ((roll-roll_lag) + (comm-comm_lag)) / extract(epoch from (time - time_lag)) )\nfrom (\n select\n time, lag(time) over w as time_lag,\n (data->>'xact_rollback')::int8 as roll, lag((data->>'xact_rollback')::int8) over w as roll_lag,\n (data->>'xact_commit')::int8 as comm, lag((data->>'xact_commit')::int8) over w as comm_lag\n FROM\n db_stats\n WHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n window w as (order by time)\n) x\nWHERE (comm > comm_lag or roll > roll_lag) and time > time_lag\nGROUP BY 1\nORDER BY 1 DESC\nLIMIT 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "xact_commit" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "0.1,1", + "timeFrom": null, + "title": "TPS (avg.)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": null, + "decimals": 1, + "description": "Queries per Second based on \"pg_stat_statements\" extension info. Last $online_interval avg. < 0.1 Error / < 1 Warning thresholds by default", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 12, + "y": 4 + }, + "hideTimeOverride": true, + "id": 16, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'DB overview Unprivileged / Developer :: QPS' panel", + "url": "/d/db-overview-developer?$__all_variables&viewPanel=15&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "cpu_load", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $online_interval),\n avg( (c - c_lag) / extract(epoch from time - time_lag)) as qps\nfrom (\n select \n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from stat_statements_calls\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and time > time_lag\ngroup by 1\norder by 1 desc\nlimit 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load_5min" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "0.1, 1", + "timeFrom": null, + "title": "QPS (avg.)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Transactions opened but where no queries are being executed.", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 18, + "y": 4 + }, + "hideTimeOverride": true, + "id": 33, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'DB overview :: Backends' panel", + "url": "/d/db-overview?$__all_variables&viewPanel=11&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "15m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "backends", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n max((data->'idleintransaction')::int)\nFROM\n backends\nWHERE\n time > now() - '$online_interval'::interval\n AND dbname = '$dbname'\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "idleintransaction" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1,5", + "timeFrom": "$online_interval", + "title": "\"Idle in TX\" count", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 1, + "description": "Size for selected DB only", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 0, + "y": 6 + }, + "hideTimeOverride": true, + "id": 25, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "DB overview :: DB size panel", + "url": "/d/db-overview?$__all_variables&viewPanel=10&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1h" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time,\n (data->'size_b')::int8\nFROM\n db_size\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nORDER BY time DESC\nLIMIT 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "postmaster_uptime_s" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": null, + "title": "DB size (last)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 1, + "description": "", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 6, + "y": 6 + }, + "hideTimeOverride": true, + "id": 23, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "DB overview :: DB size panel", + "url": "/d/db-overview?$__all_variables&viewPanel=10&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1h" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n time,\n last_value-first_value as size_diff\nfrom (\n select\n time,\n first_value((data->>'size_b')::int8) over w,\n last_value((data->>'size_b')::int8) over w\n from\n db_size\n where \n dbname = '$dbname' and $__timeFilter(time)\n window w as (\n order by time rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\n )\n) x\norder by 1 desc\nlimit 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "postmaster_uptime_s" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": null, + "title": "DB size change (diff.)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": null, + "decimals": 1, + "description": "Assumes get_psutil_disk wrapper being installed. 10/50GB Warn/Error thresholds by default", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 12, + "y": 6 + }, + "hideTimeOverride": true, + "id": 24, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "Open 'System Stats :: Free disk ' panel", + "url": "/d/system-stats?$__all_variables&viewPanel=8&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1h" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_changes", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n time,\n (data->'free')::int8\nfrom\n psutil_disk\nwhere \n dbname = '$dbname' and $__timeFilter(time)\n and tag_data->>'dir_or_tablespace' = 'data_directory'\norder by 1 desc\nlimit 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "event" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "10000000,50000000", + "timeFrom": null, + "title": "DATADIR disk space left", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 1, + "description": "Last $online_interval avg. based on \"pg_stat_statements\" info. > 0.5s Warning / > 5s Error thresholds by default", + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 18, + "y": 6 + }, + "hideTimeOverride": true, + "id": 17, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'DB overview:: Query runtime' panel", + "url": "/d/db-overview?$__all_variables&viewPanel=12&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "cpu_load", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n$__timeGroup(time, $online_interval),\navg(case when c = c_lag then 0 else (tt-tt_lag)::numeric / (c-c_lag) end) as \"avg_query_runtime\"\nfrom (\n\n select \n (data->>'total_time')::float8 as tt, lag((data->>'total_time')::float8) over w as tt_lag,\n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n time\n from stat_statements_calls\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere c >= c_lag and tt >= tt_lag\ngroup by 1\norder by 1 desc\nlimit 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load_5min" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "500, 5000", + "timeFrom": null, + "title": "Query runtime (avg.)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Server configuration changes. >1 Warn thresholds by default", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 0, + "y": 8 + }, + "hideTimeOverride": true, + "id": 22, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'DDL / Config change events' dash", + "url": "/d/change-events?$__all_variables" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1h" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_changes", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n count(*)\nFROM\n configuration_changes\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "event" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1", + "timeFrom": null, + "title": "Config change events", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Number of DDL changes i.e. new or changed tables, columns, functions etc. >1 Warn thresholds by default", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 6, + "y": 8 + }, + "hideTimeOverride": true, + "id": 20, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'DDL / Config change events' dash", + "url": "/d/change-events?$__all_variables" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1h" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_changes", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n count(*)\nFROM\n table_changes\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "event" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1", + "timeFrom": null, + "title": "Table changes", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Based on pg_stat_archiver. N/A when archiving is not enabled.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 12, + "y": 8 + }, + "hideTimeOverride": true, + "id": 19, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "365d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "archiver", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time,\n (data->'is_failing_int')::int\nFROM\n archiver\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nORDER BY time DESC\nLIMIT 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "is_failing_int" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1,2", + "timeFrom": "$online_interval", + "title": "WAL archiving status", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + }, + { + "op": "=", + "text": "OK", + "value": "0" + }, + { + "op": "=", + "text": "FAILING", + "value": "1" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 1, + "description": "Transaction Log Folder size. Assumes get_wal_size helper being installed. 1/10 GB Warn / 50 Error thresholds by default", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 18, + "y": 8 + }, + "hideTimeOverride": true, + "id": 14, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "DB overview :: WAL panel", + "url": "/d/db-overview?$__all_variables&viewPanel=10&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "cpu_load", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time,\n (data->'wal_size_b')::float8\nFROM\n wal_size\nWHERE\n time > now() - '$online_interval'::interval\n AND dbname = '$dbname'\nORDER BY time DESC\nLIMIT 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load_5min" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1000000000,10000000000", + "timeFrom": null, + "title": "WAL folder size", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Invalid indexes should be dropped as they're updated on DML operations but cannot be used for speeding up queries. In case multiple indexes of same structure Postgres will use the smallest so the others can be dropped. 'N/A' displayed both when no duplicate indexes or no metrics data for index_stats.", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 0, + "y": 10 + }, + "hideTimeOverride": true, + "id": 29, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "\"Index overview\" dashboard", + "url": "/d/index-overview" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "WITH q_duplicates AS (\nSELECT\n coalesce(sum(count-1), 0) as extra\nFROM (\nSELECT\n tag_data->>'table_full_name' as table_full_name,\n data->>'index_def' as index_def,\n count(*)\nFROM\n index_stats\nWHERE\n time = (select max(time) from index_stats where $__timeFilter(time) AND dbname = '$dbname')\n AND dbname = '$dbname'\nGROUP BY 1, 2\nHAVING count(*) > 1\n) x\n)\nSELECT\n 0 as time,\n sum((data->'is_invalid_int')::int) + (select extra from q_duplicates)\nFROM\n index_stats\nWHERE\n time = (select max(time) from index_stats where $__timeFilter(time) AND dbname = '$dbname')\n AND dbname = '$dbname'\nGROUP BY 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_relation_size_b" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1,2", + "timeFrom": null, + "title": "Invalid / duplicate indexes", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "i.e. AUTOVACUUM disabled globally or per table.", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 6, + "y": 10 + }, + "hideTimeOverride": true, + "id": 30, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n sum(count)\nFROM (\n\nSELECT * FROM (\n\nSELECT\n case when data->>'autovacuum' = 'off' then 1 else 0 end as count\nFROM\n settings\nWHERE\n dbname = '$dbname'\n AND time > now() - '1d'::interval\nORDER BY time DESC LIMIT 1\n\n) a\n\nUNION ALL\n\nSELECT * FROM (\n\nSELECT\n count(*)\nFROM\n table_stats\nWHERE\n dbname = '$dbname'\n AND time = (select max(time) from table_stats where dbname = '$dbname')\n AND (data->'no_autovacuum')::int = 1\n\n) b\n\n) x;", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_relation_size_b" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1,10", + "timeFrom": null, + "title": "Autovacuum issues", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Checkpoints should normally be performed by the background \"Checkpointer\" process. If seeing high numbers then adjusting server configuration is recommended.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 12, + "y": 10 + }, + "hideTimeOverride": true, + "id": 26, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "Checkpointer / Bgwriter / Block IO Stats", + "url": "/d/checkpointer-bgwriter-stats?$__all_variables" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "365d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "bgwriter", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n coalesce(sum(req - req_lag), 0)\nFROM (\nSELECT\n (data->'checkpoints_req')::int as req,\n lag((data->'checkpoints_req')::int) over(order by time) as req_lag\nFROM\n bgwriter\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n) x\nWHERE req > req_lag\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "checkpoints_req" + ], + "type": "field" + }, + { + "params": [], + "type": "spread" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1,2", + "timeFrom": null, + "title": "Checkpoints requested", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "diff" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 1, + "description": "Estimated extra space held by tables (excluding indexes) that would disappear after \"compacting\".", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 18, + "y": 10 + }, + "hideTimeOverride": true, + "id": 27, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time,\n (data->'approx_table_bloat_b')::int8\nFROM\n table_bloat_approx_summary_sql\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nORDER BY 1 DESC\nLIMIT 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_relation_size_b" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "1d", + "title": "Approx. table bloat", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 1, + "description": "Transaction logs generation velocity.", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 0, + "y": 12 + }, + "hideTimeOverride": true, + "id": 36, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "DB overview :: WAL panel", + "url": "/d/db-overview?$__all_variables&viewPanel=10&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "wal", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n 0 as time,\n avg((wal-wal_lag) / (etime-lag_etime)) as \"WAL rate\"\nfrom (\n select \n (data->>'xlog_location_b')::int8 as wal, lag((data->>'xlog_location_b')::int8) over w as wal_lag,\n extract(epoch from time) as etime,\n lag(extract(epoch from time)) over w as lag_etime,\n time\n from wal\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere wal >= wal_lag and etime > lag_etime", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "xlog_location_b" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1000000,16000000", + "timeFrom": null, + "title": "WAL per second (avg.)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 1, + "description": "Postgres writes temporary files to disk when sorting / grouping big amounts of data that doesn't fit into \"work_mem\". 10/100 MB Warn / Error by default", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 6, + "y": 12 + }, + "hideTimeOverride": true, + "id": 37, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "DB overview", + "url": "/d/db-overview?$__all_variables&viewPanel=19&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n 0 as time,\n avg(case when tb-tb_lag > 0 then (tb-tb_lag) / (etime - lag_etime) else 0 end) as \"Temp bytes (1h rate)\"\nfrom (\n select \n (data->>'temp_bytes')::int8 as tb, lag((data->>'temp_bytes')::int8) over w as tb_lag,\n (extract(epoch from time))::int8 as etime,\n (lag(extract(epoch from time)) over w)::int8 as lag_etime\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere tb >= tb_lag and etime > lag_etime\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "temp_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "10000000,100000000", + "timeFrom": null, + "title": "Temp. bytes per second (avg.)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Long running AUTOVACUUM processes can hint at workload or configuration problems", + "format": "dtdurations", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 12, + "y": 12 + }, + "hideTimeOverride": true, + "id": 35, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "15m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n max((data->'longest_autovacuum_seconds')::int8) as longest_autovacuum_seconds\nFROM\n backends\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "seconds_since_last_vacuum" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "60,300", + "timeFrom": null, + "title": "Longest AUTOVACUUM duration", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 1, + "description": "Frequent sequential scans on bigger tables should usually be avoided. Warn / Error thresholds: 10 / 100", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 18, + "y": 12 + }, + "hideTimeOverride": true, + "id": 39, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'Tables Top :: Most scans' panel", + "url": "/d/tables-top?$__all_variables&viewPanel=4&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "5m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time,\n sum((scan-scan_lag) * 60 / extract(epoch from (time-time_lag)))\nFROM (\n SELECT\n time, lag(time) over w as time_lag,\n (data->'seq_scan')::int8 as scan, lag((data->'seq_scan')::int8) over w as scan_lag\n FROM\n table_stats\n WHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n AND (data->'table_size_b')::int8 > 100000000\n WINDOW w AS (partition by tag_data->>'table_full_name' order by time)\n) x\nWHERE scan > scan_lag and time > time_lag\nGROUP BY 1\nORDER BY 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "seq_scan" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "1m" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "10,100", + "timeFrom": null, + "title": "Seq. scans on >100 MB tables per minute (avg.)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "On non-temporary tables", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 0, + "y": 14 + }, + "hideTimeOverride": true, + "id": 38, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'Tables Top :: Inserts' panel", + "url": "/d/tables-top?$__all_variables&viewPanel=6&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "5m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n time,\n avg(((ins-ins_lag) * 60) / extract(epoch from time - time_lag))\nfrom (\n select \n (data->>'tup_inserted')::int8 as ins, lag((data->>'tup_inserted')::int8) over w as ins_lag,\n time, lag(time) over w as time_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere ins >= ins_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_updated" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "1m" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": null, + "title": "INSERT-s per minute (avg.)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "On non-temporary tables", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 6, + "y": 14 + }, + "hideTimeOverride": true, + "id": 40, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'Tables Top :: Updates' panel", + "url": "/d/tables-top?$__all_variables&viewPanel=7&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "5m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n time,\n avg(((upd-upd_lag) * 60) / extract(epoch from time - time_lag))\nfrom (\n select \n (data->'tup_updated')::int8 as upd, lag((data->'tup_updated')::int8) over w as upd_lag,\n time, lag(time) over w as time_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere upd >= upd_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_updated" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "1m" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": null, + "title": "UPDATE-s per minute (avg.)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "On non-temporary tables", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 12, + "y": 14 + }, + "hideTimeOverride": true, + "id": 41, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'Tables Top :: Deletes' panel", + "url": "/d/tables-top?$__all_variables&viewPanel=8&fullscreen" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "5m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n time,\n avg(((del-del_lag) * 60) / extract(epoch from time - time_lag))\nfrom (\n select \n (data->'tup_deleted')::int8 as del, lag((data->'tup_deleted')::int8) over w as del_lag,\n time, lag(time) over w as time_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere del >= del_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_updated" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "1m" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": null, + "title": "DELETE-s per minute (avg.)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Based on db_stats.backup_duration_s / pg_backup_start_time() [9.3+] function. N/A if no backup in progress.", + "format": "dtdurations", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 18, + "y": 14 + }, + "hideTimeOverride": true, + "id": 42, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "backends", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n max((data->'backup_duration_s')::int8)\nFROM\n db_stats\nWHERE\n $__timeFilter(time)\n and dbname = '$dbname'", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "max_xmin_age_tx" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "3600,36000", + "timeFrom": "$online_interval", + "title": "Backup duration", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 1, + "description": "Max. age since last VACUUM FREEZE. Could hint at problems when going into 100s of millions", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 0, + "y": 16 + }, + "hideTimeOverride": true, + "id": 47, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "365d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication_slots", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n max((data->'tx_freeze_age')::int8)\nFROM\n table_stats\nWHERE\n $__timeFilter(time)\n and dbname = '$dbname'", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "max_xmin_age_tx" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "100000000,500000000", + "timeFrom": null, + "title": "Max. table FREEZE age", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "In TX. Holds back VACUUM", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 6, + "y": 16 + }, + "hideTimeOverride": true, + "id": 43, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'Replication lag' dashboard", + "url": "/d/replication?$__all_variables" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "365d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication_slots", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n 0 as time,\n max(xmin) from (\nSELECT\n max((data->'xmin_age_tx')::int8) as xmin\nFROM\n replication_slots\nWHERE\n time = (select max(time) from replication_slots where time > now() - '$online_interval'::interval AND dbname = '$dbname')\n and dbname = '$dbname' \nUNION ALL\nSELECT\n max((data->'max_xmin_age_tx')::int8) as xmin\nFROM\n backends\nWHERE\n time = (select max(time) from backends where time > now() - '$online_interval'::interval AND dbname = '$dbname')\n and dbname = '$dbname'\n) x", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "max_xmin_age_tx" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "10000,100000", + "timeFrom": null, + "title": "Max. XMIN horizon age", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "#bf1b00", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Inactive repl. slots accumulate WAL files until disk space runs out. Re-animate the replica or use pg_drop_replication_slot() to remove the slot so that Postgres can delete the un-needed WALs.", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 12, + "y": 16 + }, + "hideTimeOverride": true, + "id": 4, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'Replication lag' dashboard", + "url": "/d/replication?$__all_variables" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication_slots", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n sum((data->'non_active_int')::int)\nFROM\n replication_slots\nWHERE\n time = (select max(time) from replication_slots where time > now() - '$online_interval'::interval AND dbname = '$dbname')\n AND dbname = '$dbname'\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "active" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1,1000", + "timeFrom": "$online_interval", + "title": "Inactive repl. slots", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "#FA6400", + "#d44a3a" + ], + "datasource": null, + "decimals": null, + "description": "\"Apply\" (data made visible to queries on replica) lag in bytes", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 6, + "x": 18, + "y": 16 + }, + "hideTimeOverride": true, + "id": 46, + "interval": null, + "links": [ + { + "targetBlank": true, + "title": "'Replication lag' dashboard", + "url": "/d/replication?$__all_variables" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication_slots", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n max((data->'replay_lag_b')::int8)\nFROM\n replication\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "active" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1000000,10000000", + "timeFrom": "$online_interval", + "title": "Max. replication lag", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "content": "Brought to you by: \"Cybertec", + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 12, + "links": [], + "mode": "html", + "options": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "content": "

The preset threshold values (green / yellow / red) should be reviewed / adjusted!

", + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 48, + "links": [], + "mode": "html", + "options": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 19, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "10m", + "value": "10m" + }, + "hide": 0, + "label": "Max. age for 'online' metrics", + "name": "online_interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "3m", + "value": "3m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": true, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + } + ], + "query": "1m,3m,5m,10m,15m", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "hidden": false, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Health-check", + "uid": "health-check", + "version": 1 + } + index-overview.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "columns": [], + "datasource": null, + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 8, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 3, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "index_def_hash", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "max_index_size", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Table details' dashboard", + "linkUrl": "/d/table-details?var-dbname=${__cell_1}&var-table_full_name=${__cell}", + "mappingType": 1, + "pattern": "table_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "count", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select \"index_def\", \"max_index_size\", \"count\" from (select mode(\"index_def\") as \"index_def\", count(distinct(\"index_full_name_val\")), max(\"index_size_b\") as \"max_index_size\" from index_stats where \"dbname\" =~ /^$dbname$/ AND $timeFilter group by \"table_full_name\", \"index_def_hash\", \"dbname\") where \"count\" > 1 group by \"table_full_name\", \"index_def_hash\", \"dbname\"", + "rawQuery": true, + "rawSql": "--select \"index_def\", \"max_index_size\", \"count\" from (select mode(\"index_def\") as \"index_def\", count(distinct(\"index_full_name_val\")), max(\"index_size_b\") as \"max_index_size\" \n-- from index_stats where \"dbname\" =~ /^$dbname$/ AND $timeFilter group by \"table_full_name\", \"index_def_hash\", \"dbname\") where \"count\" > 1 \n--group by \"table_full_name\", \"index_def_hash\", \"dbname\"\nSELECT\n dbname,\n tag_data->>'table_full_name' as table_full_name,\n data->>'index_def' as index_def,\n max((data->'index_size_b')::int8) as max_index_size,\n count(*)\nFROM\n index_stats\nWHERE\n time in (select max(time) from index_stats where $__timeFilter(time) AND dbname in ($dbname) group by dbname)\n AND dbname in ($dbname)\nGROUP BY\n 1, 2, 3\nHAVING\n count(*) > 1", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": "15m", + "title": "Duplicate indexes", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Before dropping indexes make sure they are not used on replicas, if having any. FYI - one could also adjust the default 0 threshold so that \"little used\" indexes are shown", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 10, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 3, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Table details' dashboard", + "linkUrl": "/d/table-details?var-dbname=${__cell_1}&var-table_full_name=${__cell}", + "mappingType": 1, + "pattern": "table_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "index_size", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select \"index_size\", \"idx_scans_from_last_reset\" from (select last(\"index_size_b\") as \"index_size\", last(\"idx_scan\") as \"idx_scans_from_last_reset\", last(\"is_pk\") as \"is_pk\" from index_stats where \"dbname\" =~ /^$dbname$/ AND $timeFilter group by \"table_full_name\", \"index_full_name\", \"dbname\") where \"idx_scans_from_last_reset\" = 0 and \"is_pk\" = false group by \"table_full_name\", \"index_full_name\", \"dbname\"", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n tag_data->>'index_full_name' as index_full_name,\n tag_data->>'table_full_name' as table_full_name,\n max((data->'index_size_b')::int8) as index_size,\n max((data->'idx_scan')::int8) as idx_scans_from_last_reset\nFROM\n index_stats\nWHERE\n time in (select max(time) from index_stats where $__timeFilter(time) AND dbname in ($dbname) group by dbname)\n AND dbname in ($dbname)\n AND (data->'idx_scan')::int8 = 0\n AND (data->'is_pk_int')::int8 = 0\n AND coalesce((data->'is_uq_or_exc')::int8, 0) = 0\nGROUP BY\n 1, 2, 3\nORDER BY\n 4 desc", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": "15m", + "title": "Unused indexes (Non-PK)", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "fontSize": "100%", + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 11, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 3, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Table details' dashboard", + "linkUrl": "/d/table-details?var-dbname=${__cell_1}&var-table_full_name=${__cell}", + "mappingType": 1, + "pattern": "table_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "index_size", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select \"index_size\" from (select last(\"index_size_b\") as \"index_size\", last(\"is_invalid_int\") as \"is_invalid_int\" from index_stats where \"dbname\" =~ /^$dbname$/ AND $timeFilter group by \"table_full_name\", \"index_full_name\", \"dbname\") where \"is_invalid_int\" = 1 group by \"table_full_name\", \"index_full_name\", \"dbname\"", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n tag_data->>'index_full_name' as index_full_name,\n tag_data->>'table_full_name' as table_full_name,\n max((data->'index_size_b')::int8) as index_size\nFROM\n index_stats\nWHERE\n time in (select max(time) from index_stats where $__timeFilter(time) AND dbname in ($dbname) group by dbname)\n AND dbname in ($dbname)\n AND (data->'is_invalid_int')::int8 = 1\nGROUP BY\n 1, 2, 3\nORDER BY\n 4 desc", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": "15m", + "title": "Invalid indexes", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 9, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 3, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Table details' dashboard", + "linkUrl": "/d/table-details?var-dbname=${__cell_1}&var-table_full_name=${__cell}", + "mappingType": 1, + "pattern": "table_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "index_size", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "idx_scans_from_last_reset", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select \"dbname\", \"index_full_name\", \"table_full_name\", top(\"index_size_b\", 20), \"idx_scans_from_last_reset\" from (select last(\"index_size_b\") as \"index_size_b\", last(\"idx_scan\") as idx_scans_from_last_reset from index_stats where \"dbname\" =~ /^$dbname$/ AND $timeFilter group by \"table_full_name\", \"index_full_name\", \"dbname\")", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n tag_data->>'index_full_name' as index_full_name,\n tag_data->>'table_full_name' as table_full_name,\n max((data->'index_size_b')::int8) as top,\n max((data->'idx_scan')::int8) as scans_from_last_reset\nFROM\n index_stats\nWHERE\n time in (select max(time) from index_stats where $__timeFilter(time) AND dbname in ($dbname) group by dbname)\n AND dbname in ($dbname)\nGROUP BY\n 1, 2, 3\nORDER BY\n 4 DESC\nLIMIT\n 20", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": "15m", + "title": "Top 20 biggest indexes", + "transform": "table", + "type": "table" + }, + { + "content": "Brought to you by: \"Cybertec", + "editable": true, + "error": false, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 6, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'index_stats' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Index overview", + "uid": "index-overview", + "version": 1 + } + postgres-version-overview.json: |- + { + "annotations": { + "list": [ + { + "$hashKey": null, + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "columns": [], + "datasource": null, + "description": "As per \"settings\" metric.", + "fontSize": "130%", + "gridPos": { + "h": 14, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [], + "options": {}, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": false + }, + "styles": [ + { + "alias": "", + "colorMode": "row", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "server_version_num", + "thresholds": [ + "90400", + "90600" + ], + "type": "number", + "unit": "none" + } + ], + "targets": [ + { + "$hashKey": null, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "sproc_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT top(\"spread\", \"function_full_name\", \"oid\", $top) FROM (SELECT spread(\"total_time\") FROM \"sproc_stats\" WHERE $timeFilter AND \"dbname\" = '$dbname' GROUP BY time(3650d), \"function_full_name\", \"oid\" fill(none) ) WHERE spread > 0", + "rawQuery": true, + "rawSql": "SELECT\n split_part(data->>'server_version'::text, ' ', 1) as server_version,\n (data->'server_version_num')::int8 as server_version_num,\n string_agg(DISTINCT dbname, ', ') as dbname \nFROM\n settings\nWHERE\n $__timeFilter(time)\n AND dbname IN ($dbname)\n AND (data->>'server_version_num')::int8 <= $lower_than_server_version_num\nGROUP BY 1, 2\nORDER BY 1", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "total_time" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Monitored DBs by version", + "transform": "table", + "type": "table" + }, + { + "content": "Brought to you by: \"Cybertec", + "datasource": null, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 8, + "links": [], + "mode": "html", + "options": {}, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": false, + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 20, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": null, + "definition": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'settings' ORDER BY 1;", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'settings' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": null, + "definition": "SELECT DISTINCT (data->'server_version_num')::int FROM settings WHERE time > now() - '24h'::interval ORDER BY 1 DESC", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "lower_than_server_version_num", + "options": [], + "query": "SELECT DISTINCT (data->'server_version_num')::int FROM settings WHERE time > now() - '24h'::interval ORDER BY 1 DESC", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Postgres Version Overview", + "uid": "postgres-version-overview", + "version": 1 + } + recommendations.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Needs \"recommendations\" metric to be enabled", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "columns": [], + "datasource": null, + "description": "More than 10 superusers is already suspicious", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n data->>'major_ver' AS pg_major_ver,\n tag_data->>'object_name' AS object_name,\n data->>'recommendation' AS recommendation,\n data->>'extra_info' AS extra_info\nFROM\n recommendations\nWHERE\n time IN (SELECT max(time) FROM recommendations WHERE $__timeFilter(time) GROUP BY dbname)\n AND dbname IN ($dbname)\n AND tag_data->>'reco_topic' = 'superuser_count'\nORDER BY\n dbname, object_name", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Overusage of SUPERUSER", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 4, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n '-' AS pg_major_ver,\n '-' AS object_name,\n 'long running queries block autovacuum and thereby cause bloat' AS recommendation,\n 'longest_query_seconds: '|| max((data->'longest_query_seconds')::int)::text AS extra_info\nFROM\n backends\nWHERE\n $__timeFilter(time)\n AND dbname IN ($dbname)\n AND (data->'longest_query_seconds')::int > 6\nGROUP BY\n dbname\nORDER BY\n max((data->'longest_query_seconds')::int) DESC NULLS LAST, dbname", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Overly long queries", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 10, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n (select data->>'server_version' from settings s where s.dbname = x.dbname order by time desc limit 1) AS pg_major_ver,\n 'autovacuum_max_workers' as object_name,\n 'if most / all autovacuum worker slots are busy or AV runs for hourse then this could hint at incorrect AV params or too much workload' as recommendation,\n 'max. active autovacuum workers: ' || max_av_workers || ' (of total ' || (select (data->'autovacuum_max_workers')::int from settings s where s.dbname = x.dbname order by time desc limit 1) || '); max. AV duration (seconds): ' || longest_autovacuum_seconds as extra_info\nFROM (\nSELECT\n dbname,\n max((data->'av_workers')::int) as max_av_workers,\n coalesce(max((data->'longest_autovacuum_seconds')::int), 0) as longest_autovacuum_seconds\nFROM\n backends\nWHERE\n $__timeFilter(time)\n AND dbname IN ($dbname)\nGROUP BY\n dbname\n) x\nWHERE\n (max_av_workers > 1 AND max_av_workers::numeric / (select (data->'autovacuum_max_workers')::int from settings s where s.dbname = x.dbname order by time desc limit 1) >= 0.6)\n OR\n (longest_autovacuum_seconds > 1800)", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Possible AUTOVACUUM problems", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 5, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n data->>'major_ver' AS pg_major_ver,\n tag_data->>'object_name' AS object_name,\n data->>'recommendation' AS recommendation,\n data->>'extra_info' AS extra_info\nFROM\n recommendations\nWHERE\n time IN (SELECT max(time) FROM recommendations WHERE $__timeFilter(time) GROUP BY dbname)\n AND dbname IN ($dbname)\n AND tag_data->>'reco_topic' = 'sprocs_wo_search_path'\nORDER BY\n dbname, object_name", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "SECURITY DEFINER without fixed \"search_path\"", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 6, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n data->>'major_ver' AS pg_major_ver,\n tag_data->>'object_name' AS object_name,\n data->>'recommendation' AS recommendation,\n data->>'extra_info' AS extra_info\nFROM\n recommendations\nWHERE\n time IN (SELECT max(time) FROM recommendations WHERE $__timeFilter(time) GROUP BY dbname)\n AND dbname IN ($dbname)\n AND tag_data->>'reco_topic' = 'default_public_schema_privs'\nORDER BY\n dbname, object_name", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "PUBLIC schemas with default privileges", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Assumes that \"pg_qualstats\" extension and superuser or according access grants on the monitored host", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 7, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n data->>'major_ver' AS pg_major_ver,\n tag_data->>'object_name' AS object_name,\n data->>'recommendation' AS recommendation,\n data->>'extra_info' AS extra_info\nFROM\n recommendations\nWHERE\n time IN (SELECT max(time) FROM recommendations WHERE $__timeFilter(time) GROUP BY dbname)\n AND dbname IN ($dbname)\n AND tag_data->>'reco_topic' = 'create_index'\nORDER BY\n dbname, object_name", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Possibly missing indexes", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Unused indexes bigger than 0.5% of total DB size", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 8, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n data->>'major_ver' AS pg_major_ver,\n tag_data->>'object_name' AS object_name,\n data->>'recommendation' AS recommendation,\n data->>'extra_info' AS extra_info\nFROM\n recommendations\nWHERE\n time IN (SELECT max(time) FROM recommendations WHERE $__timeFilter(time) GROUP BY dbname)\n AND dbname IN ($dbname)\n AND tag_data->>'reco_topic' = 'drop_index'\nORDER BY\n dbname, object_name", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Unused indexes", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "A lot of space / performance can be won on replacing columns that contain mostly NULL-s but are fully indexed with partial indexes, leaving out NULL-s", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 12, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n data->>'major_ver' AS pg_major_ver,\n tag_data->>'object_name' AS object_name,\n data->>'recommendation' AS recommendation,\n data->>'extra_info' AS extra_info\nFROM\n recommendations\nWHERE\n time IN (SELECT max(time) FROM recommendations WHERE $__timeFilter(time) GROUP BY dbname)\n AND dbname IN ($dbname)\n AND tag_data->>'reco_topic' = 'partial_index_candidates'\nORDER BY\n dbname, object_name", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Partial index suggestions", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 2, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n data->>'major_ver' AS pg_major_ver,\n tag_data->>'object_name' AS object_name,\n data->>'recommendation' AS recommendation,\n data->>'extra_info' AS extra_info\nFROM\n recommendations\nWHERE\n time IN (SELECT max(time) FROM recommendations WHERE $__timeFilter(time) GROUP BY dbname)\n AND dbname IN ($dbname)\n AND tag_data->>'reco_topic' = 'overly_nested_views'\nORDER BY\n dbname, object_name", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Deeply nested views", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "More often than checkpoint_timeout / 2. To improve increase checkpoint_timeout or max_wal_size, which will reduce total IO amount done due to the way how Full Page Writes are handled.", + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 11, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n (select data->>'server_version' from settings s where s.dbname = y.dbname order by s.time desc limit 1) as pg_major_ver,\n 'global server settings' as object_name,\n 'Increase checkpoint_timeout or max_wal_size for more infrequent checkpoints, thereby reducing total IO' AS recommendation,\n 'avg. seconds between checkpoint requests: ' || avg_seconds_between_checkpoints_reqs::int8 as extra_info\nFROM (\nSELECT\n dbname,\n round((extract(epoch from max_time - min_time) / (max_req - min_req))::numeric, 1) as avg_seconds_between_checkpoints_reqs\nFROM (\n SELECT\n dbname,\n min(time) as min_time,\n max(time) as max_time,\n min((data->'checkpoints_req')::int8) as min_req,\n max((data->'checkpoints_req')::int8) as max_req\n FROM\n bgwriter\n WHERE\n $__timeFilter(time)\n AND dbname IN ($dbname)\nGROUP BY\n dbname\nHAVING\n max((data->'checkpoints_req')::int8) > min((data->'checkpoints_req')::int8)\n) x\n) y\nWHERE\n avg_seconds_between_checkpoints_reqs < (select extract(epoch from (data->>'checkpoint_timeout')::interval) / 2.0 from settings s where s.dbname = y.dbname order by s.time desc limit 1)\nORDER BY\n avg_seconds_between_checkpoints_reqs\nLIMIT\n 25", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Checkpoints requested too often", + "transform": "table", + "type": "table" + }, + { + "content": "Brought to you by: \"Cybertec", + "datasource": null, + "description": "", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 9, + "mode": "html", + "options": {}, + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n data->>'major_ver' AS pg_major_ver,\n tag_data->>'object_name' AS object_name,\n data->>'recommendation' AS recommendation,\n data->>'extra_info' AS extra_info\nFROM\n recommendations\nWHERE\n time IN (SELECT max(time) FROM recommendations WHERE $__timeFilter(time) GROUP BY dbname)\n AND dbname IN ($dbname)\n AND tag_data->>'reco_topic' = 'drop_index'\nORDER BY\n dbname, object_name", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 20, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": null, + "definition": "SELECT DISTINCT dbname FROM recommendations WHERE time > now() - '2d'::interval ORDER BY 1;", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM recommendations WHERE time > now() - '2d'::interval ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Recommendations", + "uid": "recommendations", + "version": 1 + } + replication.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "#bf1b00", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Based on last 5m data. > 1 ~ Error. Causes WAL accumulation on the master and possibly autovacuum inefficiencies", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 0 + }, + "hideTimeOverride": true, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication_slots", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select sum(\"last\") from (select last(\"non_active_int\") FROM \"replication_slots\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter group by \"dbname\", \"slot_name\")", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n sum((data->'non_active_int')::int)\nFROM\n replication_slots\nWHERE\n time in (select max(time) from replication_slots where $__timeFilter(time) AND dbname in ($dbname) group by dbname)\n AND dbname in ($dbname)\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "active" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1,1000", + "timeFrom": "5m", + "title": "Inactive repl. slots", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "#bf1b00", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Based on last 5m data. FYI - no thresholds defined", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 3, + "y": 0 + }, + "hideTimeOverride": true, + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication_slots", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select count(\"active\") from (select last(\"active\") as \"active\" FROM \"replication_slots\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter group by \"dbname\", \"slot_name\") WHERE \"active\" = true", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n count(*) filter (where data->>'non_active_int' = '0')\nFROM\n replication_slots\nWHERE\n time in (select max(time) from replication_slots where $__timeFilter(time) AND dbname in ($dbname) group by dbname)\n AND dbname in ($dbname)\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "active" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1,1000", + "timeFrom": "5m", + "title": "Active repl. slots", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Data is available only on connected replicas. No thresholds by default", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 6, + "y": 0 + }, + "hideTimeOverride": true, + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select count(\"last\") from (SELECT last(\"state\") FROM \"replication\" WHERE (\"dbname\" =~ /^$dbname$/) AND $timeFilter GROUP BY time(1d), dbname, client_info fill(none)) where \"last\" = 'streaming'", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n count(*)\nFROM\n replication\nWHERE\n time in (select max(time) from replication where $__timeFilter(time) AND dbname in ($dbname) group by dbname)\n AND dbname in ($dbname)\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "replay_lag_b" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "100000000,1000000000", + "timeFrom": "5m", + "title": "Active replicas", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 0, + "description": "Data is available only on connected replicas. No thresholds by default", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 9, + "y": 0 + }, + "hideTimeOverride": true, + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select sum(\"last\") from (SELECT last(\"is_sync_int\") FROM \"replication\" WHERE (\"dbname\" =~ /^$dbname$/) AND $timeFilter GROUP BY time(1d), dbname, client_info fill(none))", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n count(*) filter (where data->>'is_sync_int' = '1') \nFROM\n replication\nWHERE\n time in (select max(time) from replication where $__timeFilter(time) AND dbname in ($dbname) group by dbname)\n AND dbname in ($dbname)\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "replay_lag_b" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "5m", + "title": "Active \"sync\" replicas", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 1, + "description": "Based on last 5m of \"pg_replication_slots.restart_lsn\" and translates to \"overhead\" WAL folder size", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 12, + "y": 0 + }, + "hideTimeOverride": true, + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication_slots", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"restart_lsn_lag_b\") FROM \"replication_slots\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY time(1d) fill(none)", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n max((data->'restart_lsn_lag_b')::int8)\nFROM\n replication_slots\nWHERE\n time in (select max(time) from replication_slots where $__timeFilter(time) AND dbname in ($dbname) group by dbname)\n AND dbname in ($dbname)\n\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "restart_lsn_lag_b" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "16000000,1000000000", + "timeFrom": "5m", + "title": "Slot max. restart_lsn lag", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 1, + "description": "Data is available only on connected replicas! Based on last 5m data.", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 15, + "y": 0 + }, + "hideTimeOverride": true, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"write_lag_b\") FROM \"replication\" WHERE $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n max((data->'write_lag_b')::int8)\nFROM\n replication\nWHERE\n time in (select max(time) from replication where $__timeFilter(time) AND dbname in ($dbname) group by dbname)\n AND dbname in ($dbname)\n\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "write_lag_b" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "100000000,1000000000", + "timeFrom": "5m", + "title": "Max. write lag", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 1, + "description": "Data is available only on connected replicas! Based on last 5m data.", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 18, + "y": 0 + }, + "hideTimeOverride": true, + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"write_lag_b\") FROM \"replication\" WHERE $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n max((data->'flush_lag_b')::int8)\nFROM\n replication\nWHERE\n time in (select max(time) from replication where $__timeFilter(time) AND dbname in ($dbname) group by dbname)\n AND dbname in ($dbname)\n\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "flush_lag_b" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "100000000,1000000000", + "timeFrom": "5m", + "title": "Max. flush lag", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": 1, + "description": "Data is available only on connected replicas! Based on last 5m data.", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 21, + "y": 0 + }, + "hideTimeOverride": true, + "id": 9, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"replay_lag_b\") FROM \"replication\" WHERE (\"dbname\" =~ /^$dbname$/) AND $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n max((data->'replay_lag_b')::int8)\nFROM\n replication\nWHERE\n time in (select max(time) from replication where $__timeFilter(time) AND dbname in ($dbname) group by dbname)\n AND dbname in ($dbname)\n\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "replay_lag_b" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "100000000,1000000000", + "timeFrom": "5m", + "title": "Max. replay lag", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Based on pg_replication_slots. Primary only", + "fill": 1, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 2, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_dbname]] slot=[[tag_slot_name]]", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "dbname" + ], + "type": "tag" + }, + { + "params": [ + "slot_name" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication_slots", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $__interval),\n dbname,\n max((data->'restart_lsn_lag_b')::int8)\nFROM\n replication_slots\nWHERE\n $__timeFilter(time)\n AND dbname in ($dbname)\nGROUP BY 1, 2\nORDER BY 1, 2\n\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "restart_lsn_lag_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Replication slot restart_lsn lag (primary extra WAL size)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Based on pg_stat_replication on primaries. Data is available only on connected replicas!", + "fill": 1, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 17, + "interval": "2m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_dbname]] [[tag_application_name]] [[tag_client_info]]", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "dbname" + ], + "type": "tag" + }, + { + "params": [ + "client_info" + ], + "type": "tag" + }, + { + "params": [ + "application_name" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"write_lag_b\") FROM \"replication\" WHERE (\"dbname\" =~ /^$dbname$/) AND $timeFilter GROUP BY time($__interval), \"dbname\", \"client_info\", \"application_name\" fill(none)", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $__interval),\n dbname,\n avg((data->'flush_lag_b')::int8)\nFROM\n replication\nWHERE\n $__timeFilter(time)\n AND dbname in ($dbname)\nGROUP BY\n 1, 2\nORDER BY\n 1, 2\n \n", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "flush_lag_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Replication flush lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Based on pg_stat_replication on primaries. Data is available only on connected replicas!", + "fill": 1, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 8, + "interval": "15m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_dbname]] [[tag_application_name]] [[tag_client_info]]", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "dbname" + ], + "type": "tag" + }, + { + "params": [ + "client_info" + ], + "type": "tag" + }, + { + "params": [ + "application_name" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $__interval),\n dbname,\n avg((data->'replay_lag_b')::int8)\nFROM\n replication\nWHERE\n $__timeFilter(time)\n AND dbname in ($dbname)\nGROUP BY\n 1, 2\nORDER BY\n 1, 2\n \n", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "replay_lag_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Replication replay lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Based on \"pg_replication_slots.xmin\". The oldest transaction that this slot needs the database to retain. VACUUM cannot remove tuples deleted by any later transaction.", + "fill": 1, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 10, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_dbname]] [[tag_slot_name]]", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "dbname" + ], + "type": "tag" + }, + { + "params": [ + "plugin" + ], + "type": "tag" + }, + { + "params": [ + "slot_name" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "replication_slots", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $__interval),\n dbname,\n avg((data->'xmin_age_tx')::int8)\nFROM\n replication_slots\nWHERE\n $__timeFilter(time)\n AND dbname in ($dbname)\nGROUP BY 1, 2\nORDER BY 1, 2", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "xmin_age_tx" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Repl. slot XMIN age (in transactions)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Brought to you by: \"Cybertec", + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 12, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": false, + "schemaVersion": 18, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric IN ('replication', 'replication_slots') ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Replication", + "uid": "replication", + "version": 1 + } + server-log-events.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Assumes local (on DB server) setup and CSVLOG output format", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": { + "fatal": "super-light-red", + "panic": "dark-purple" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": null, + "description": "Based on 'server_log_event_counts' metric. Assumes local (on DB server) setup and CSVLOG output format", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n sum((data->'debug')::int8) as debug,\n sum((data->'log')::int8) as log,\n sum((data->'notice')::int8) as notice, \n sum((data->'warning')::int8) as warning,\n sum((data->'error')::int8) as error,\n sum((data->'fatal')::int8) as fatal,\n sum((data->'panic')::int8) as panic\nFROM\n server_log_event_counts\nWHERE\n $__timeFilter(time)\n AND dbname in ($dbname)\nGROUP BY\n 1\nORDER BY\n 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Server log events (DB)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "fatal": "super-light-red", + "panic": "dark-purple" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": null, + "description": "Based on 'server_log_event_counts' metric. Assumes local (on DB server) setup and CSVLOG output format", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 3, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n sum((data->'debug_total')::int8) as debug,\n sum((data->'log_total')::int8) as log,\n sum((data->'notice_total')::int8) as notice, \n sum((data->'warning_total')::int8) as warning,\n sum((data->'error_total')::int8) as error,\n sum((data->'fatal_total')::int8) as fatal,\n sum((data->'panic_total')::int8) as panic\nFROM\n server_log_event_counts\nWHERE\n $__timeFilter(time)\n AND dbname in ($dbname)\nGROUP BY\n 1\nORDER BY\n 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Server log events (whole instance)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": null, + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 5, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": null, + "pattern": "/ERROR|FATAL/", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Go to DB overview dash", + "linkUrl": "/d/db-overview?var-dbname=${__cell}", + "mappingType": 1, + "pattern": "dbname", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n sum((data->'error')::int8) as \"ERROR\",\n sum((data->'error_total')::int8) as \"ERROR_TOTAL\",\n sum((data->'fatal')::int8) as \"FATAL\", \n sum((data->'fatal_total')::int8) as \"FATAL_TOTAL\"\nFROM\n server_log_event_counts\nWHERE\n $__timeFilter(time)\n AND dbname in ($dbname)\nGROUP BY\n 1\nORDER BY\n 2 DESC\nLIMIT\n 20\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top ERROR generating DBs", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 6, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": null, + "pattern": "/ERROR|FATAL/", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Go to DB overview dash", + "linkUrl": "/d/db-overview?var-dbname=${__cell}", + "mappingType": 1, + "pattern": "dbname", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n dbname,\n sum((data->'fatal_total')::int8) as \"FATAL_TOTAL\",\n sum((data->'fatal')::int8) as \"FATAL\",\n sum((data->'error_total')::int8) as \"ERROR_TOTAL\",\n sum((data->'error')::int8) as \"ERROR\"\nFROM\n server_log_event_counts\nWHERE\n $__timeFilter(time)\n AND dbname in ($dbname)\nGROUP BY\n 1\nORDER BY\n 2 DESC\nLIMIT\n 20\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Top FATAL generating instances", + "transform": "table", + "type": "table" + }, + { + "content": "Brought to you by: \"Cybertec", + "datasource": null, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 8, + "mode": "html", + "options": {}, + "timeFrom": null, + "timeShift": null, + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 20, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": null, + "definition": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'server_log_event_counts' ORDER BY 1;", + "hide": 0, + "includeAll": true, + "index": -1, + "label": null, + "multi": true, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'server_log_event_counts' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "5m", + "value": "5m" + }, + "hide": 0, + "label": null, + "name": "agg_interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1m,5m,10m,15m,30m,1h,6h,12h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Server Log Events", + "uid": "server-log-events", + "version": 1 + } + sessions-overview.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "interval": "", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max(((c - c_lag) + (r - r_lag)) / extract(epoch from time - time_lag)) as \"TPS\"\nfrom (\n select \n (data->>'xact_commit')::int8 as c, lag((data->>'xact_commit')::int8) over w as c_lag,\n (data->>'xact_rollback')::int8 as r, lag((data->>'xact_rollback')::int8) over w as r_lag,\n time, lag(time) over w as time_lag\n from db_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and r >= r_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((c - c_lag) / extract(epoch from time - time_lag)) as \"QPS\"\nfrom (\n select \n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from stat_statements_calls\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and time > time_lag\ngroup by 1\norder by 1\n", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Max. TPS / QPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "avg_query": "super-light-purple", + "longest_query": "dark-orange" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->'longest_query_seconds')::int8) as longest_query,\n max((data->'avg_query_seconds')::float) as avg_query\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Longest query duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "longest_waiting": "dark-red", + "longest_waiting_seconds": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->'longest_waiting_seconds')::int8) as longest_waiting,\n max((data->'avg_waiting_seconds')::float) as avg_waiting\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Longest wait duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "avg_tx": "light-blue", + "longest_tx": "dark-purple", + "longest_tx_seconds": "dark-purple" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->'longest_tx_seconds')::int8) as longest_tx,\n max((data->'avg_tx_seconds')::float) as avg_tx\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Longest TX duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "longest_session_seconds": "dark-orange" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->'longest_session_seconds')::int8) as longest_session,\n max((data->'avg_session_seconds')::float) as avg_session\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Longest session duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "longest_autovacuum_seconds": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 30 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "longest_autovacuum_seconds", + "yaxis": 1 + }, + { + "alias": "max_av_workers", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->'longest_autovacuum_seconds')::int8) as longest_autovacuum_seconds\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->'av_workers')::int8) as max_av_workers\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Longest Autovacuum duration / # AV workers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "Max. value per $agg_interval", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->'active')::int8) as active,\n max((data->'total')::int8) as total_db,\n max((data->'idleintransaction')::int8) as idleintransaction,\n max((data->'waiting')::int8) as waiting\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Sessions by state", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max_connections": "dark-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n max((data->'instance_total')::int8) as instance_total,\n max((data->'max_connections')::int8) as max_connections\nfrom backends\nwhere dbname = '$dbname' and $__timeFilter(time)\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Instance total connections", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Brought to you by: \"Cybertec", + "datasource": null, + "gridPos": { + "h": 3, + "w": 9, + "x": 0, + "y": 48 + }, + "id": 13, + "mode": "html", + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": false, + "rawSql": "SELECT\n $__time(time_column),\n value1\nFROM\n metric_table\nWHERE\n $__timeFilter(time_column)\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 22, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": "", + "value": "" + }, + "datasource": null, + "definition": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'backends' ORDER BY 1;", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'backends' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "tags": [], + "text": "5m", + "value": "5m" + }, + "hide": 0, + "label": null, + "name": "agg_interval", + "options": [ + { + "selected": false, + "text": "1s", + "value": "1s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1s,1m,5m,10m,15m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Sessions overview", + "uid": "sessions-overview", + "variables": { + "list": [] + }, + "version": 1 + } diff --git a/helm/pgwatch/templates/grafana-deployment.yaml b/helm/pgwatch/templates/grafana-deployment.yaml new file mode 100644 index 0000000..05acc48 --- /dev/null +++ b/helm/pgwatch/templates/grafana-deployment.yaml @@ -0,0 +1,118 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + application: pgwatch + pgwatch.pods.role: grafana + name: grafana +spec: + replicas: 1 + selector: + matchLabels: + application: pgwatch + pgwatch.pods.role: grafana + strategy: + type: Recreate + template: + metadata: + labels: + application: pgwatch + pgwatch.pods.role: grafana + spec: + containers: + - env: + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "true" + - name: GF_AUTH_ANONYMOUS_ORG_ROLE + value: Admin + - name: GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH + value: /var/lib/grafana/dashboards/1-global-db-overview.json + - name: GF_DATABASE_HOST + value: postgres-svc:5432 + - name: GF_DATABASE_NAME + value: pgwatch_grafana + - name: GF_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: pgwatch-postgresql-secret-pgwatch + key: password + - name: GF_DATABASE_SSL_MODE + value: disable + - name: GF_DATABASE_TYPE + value: postgres + - name: GF_DATABASE_USER + valueFrom: + secretKeyRef: + name: pgwatch-postgresql-secret-pgwatch + key: username + - name: GF_INSTALL_PLUGINS + value: marcusolsson-treemap-panel + image: grafana/grafana:10.4.7 + name: grafana + ports: + - containerPort: 3000 + protocol: TCP + volumeMounts: + - mountPath: /etc/grafana/provisioning/datasources/pg_ds.yml + name: grafana-datasources + subPath: pg_ds.yml + - mountPath: /etc/grafana/provisioning/dashboards/pg_db.yml + name: grafana-config + subPath: pg_db.yml + - mountPath: /var/lib/grafana/dashboards + name: grafana-dashboards + restartPolicy: Always + volumes: + - configMap: + items: + - key: postgres_datasource.yml + path: pg_ds.yml + name: grafana-datasources + name: grafana-datasources + - configMap: + items: + - key: postgres_dashboard.yml + path: pg_db.yml + name: grafana-config + name: grafana-config + - name: grafana-dashboards + projected: + sources: + - configMap: + name: grafana-dashboards-basics + optional: true + - configMap: + name: grafana-dashboards-others + optional: true + defaultMode: 416 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + annotations: + use-subpath: "true" + labels: + application: pgwatch + name: grafana-datasources +data: + postgres_datasource.yml: "apiVersion: 1\n\ndatasources:\n- name: pg-metrics\n type: postgres\n url: postgres-svc:5432\n access: proxy\n password: pgwatchadmin\n user: pgwatch\n database: pgwatch_metrics\n basicAuth: false\n isDefault: true\n jsonData:\n sslmode: disable\n postgresVersion: 1700 \n version: 1\n editable: true\n" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + application: pgwatch + name: grafana-config +data: + postgres_dashboard.yml: | + apiVersion: 1 + + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + updateIntervalSeconds: 10 #how often Grafana will scan for changed dashboards + options: + path: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/helm/pgwatch/templates/grafana_dashboard_others.yaml b/helm/pgwatch/templates/grafana_dashboard_others.yaml new file mode 100644 index 0000000..24f3f78 --- /dev/null +++ b/helm/pgwatch/templates/grafana_dashboard_others.yaml @@ -0,0 +1,22971 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + application: pgwatch + name: grafana-dashboards-others +data: + 1-global-db-overview.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "description": "Avg. of all DBs over the whole selected period", + "editable": true, + "error": false, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": "15m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "TPS", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select mean(tx) from (SELECT non_negative_derivative(mean(\"xact_commit\"), 1s) + non_negative_derivative(mean(\"xact_rollback\"), 1s) as tx FROM \"db_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY dbname, time(5m) fill(none))", + "rawQuery": true, + "rawSql": "select\n time,\n sum(avg) as tps\nfrom (\nselect\n $__timeGroup(time, $__interval),\n dbname,\n avg(((c - c_lag) + (r - r_lag)) / extract(epoch from time - time_lag))\nfrom (\n select\n (data->>'xact_commit')::int8 as c, lag((data->>'xact_commit')::int8) over w as c_lag,\n (data->>'xact_rollback')::int8 as r, lag((data->>'xact_rollback')::int8) over w as r_lag,\n time, lag(time) over w as time_lag,\n dbname\n from db_stats\n where dbname in ($dbname) and $__timeFilter(time)\n window w as (partition by dbname order by time) \n) x\nwhere c >= c_lag and r >= r_lag and time > time_lag\ngroup by 1, 2\n) y\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": null, + "title": "TPS", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": 1, + "description": "Avg. of all DBs over the whole selected period", + "editable": true, + "error": false, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 0 + }, + "height": "150", + "id": 2, + "interval": "15m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "QPS", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(calls) FROM (SELECT non_negative_derivative(mean(\"calls\"), 1s) AS calls FROM \"stat_statements_calls\" WHERE $timeFilter AND \"dbname\" =~ /^$dbname/ GROUP BY dbname, time(5m) fill(none))", + "rawQuery": true, + "rawSql": "select\n time,\n sum(avg) as qps\nfrom (\nselect\n $__timeGroup(time, $__interval),\n dbname,\n avg((c - c_lag) / extract(epoch from time - time_lag))\nfrom (\n select \n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n time, lag(time) over w as time_lag,\n dbname\n from stat_statements_calls\n where dbname in ($dbname) and $__timeFilter(time)\n window w as (partition by dbname order by time) \n) x\nwhere c >= c_lag and time > time_lag\ngroup by 1, 2\n) y\ngroup by 1\norder by 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": null, + "title": "QPS", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "description": "Avg. of all DBs over the whole selected period for regular (>10 calls) queries", + "editable": true, + "error": false, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 0 + }, + "height": "150", + "id": 3, + "interval": "15m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "avg_query_runtime_per_db", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select mean(runtime) from (select non_negative_derivative(mean(\"total_time\")) / non_negative_derivative(mean(\"calls\")) as runtime from \"stat_statements\" WHERE \"dbname\" =~ /^$dbname/ AND $timeFilter AND calls > 100 GROUP BY dbname, time($__interval) fill(none))", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $__interval),\n avg((tt-tt_lag)::numeric / (c-c_lag))\nfrom (\n select \n (data->>'total_time')::float8 as tt, lag((data->>'total_time')::float8) over w as tt_lag,\n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n time, dbname\n from stat_statements_calls\n where dbname in ($dbname) and $__timeFilter(time)\n window w as (partition by dbname order by time)\n) x\nwhere c > c_lag and tt >= tt_lag and c > 100\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Query runtime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": 1, + "description": "Sum of all DB changes over the whole selected period", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 12, + "y": 0 + }, + "height": "150", + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "db_size_change_last_hour", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select sum(diff) from (SELECT last(\"size_b\") - first(\"size_b\") as diff FROM \"db_size\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY dbname, time(3650d) fill(none))", + "rawQuery": true, + "rawSql": "select\n time,\n sum(last_db_size) as total_size\nfrom (\nselect\n time,\n dbname,\n max(last_value) as last_db_size\nfrom (\n select\n date_trunc('hour', time) as time,\n dbname,\n last_value((data->'size_b')::int8) over w\n from\n db_size\n where \n dbname in ($dbname)\n and $__timeFilter(time)\n window w as (\n partition by dbname, date_trunc('hour', time) order by time rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING\n )\n) x\ngroup by 1, 2\n) y\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Total DB size ch.", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "diff" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": 1, + "description": "Sum of all DBs. Based on a SQL based approximate.", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 16, + "y": 0 + }, + "height": "150", + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "Bloat size", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_bloat_approx_summary", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select sum(bloat) from (SELECT last(\"approx_free_space_b\") - first(\"approx_free_space_b\") as bloat FROM \"table_bloat_approx_summary\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY dbname, time(3650d) fill(none))", + "rawQuery": true, + "rawSql": "select\n $__unixEpochFrom() as time,\n sum(bloat) from\n(\n\n select distinct on (dbname)\n dbname,\n time,\n (data->'approx_table_bloat_b')::int8 as bloat\n from\n table_bloat_approx_summary_sql\n where \n dbname in ($dbname)\n and $__timeFilter(time)\n order by\n dbname, time\n\n) a \n\nunion all\n\nselect\n $__unixEpochTo() as time,\n sum(bloat)\nfrom (\n\n select distinct on (dbname)\n dbname,\n time,\n (data->'approx_table_bloat_b')::int8 as bloat\n from\n table_bloat_approx_summary_sql\n where \n dbname in ($dbname)\n and $__timeFilter(time)\n order by\n dbname, time desc\n \n) b", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "approx_free_space_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Total table bloat ch. (approx.)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "diff" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": 1, + "description": "Avg. of all selected DB-s for the whole period", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 20, + "y": 0 + }, + "height": "150", + "id": 6, + "interval": "15m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $__interval),\n avg((data->'load_5min')::float8)\nFROM\n cpu_load\nWHERE\n $__timeFilter(time)\n AND dbname in ($dbname)\nGROUP BY 1\nORDER BY 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "CPU Load Avg.", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + "load_5": "#BA43A9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 2 + }, + "hiddenSeries": false, + "id": 18, + "interval": "15m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_dbname", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "stat_statements", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select non_negative_derivative(mean(\"xact_commit\")) + non_negative_derivative(mean(\"xact_rollback\")) from \"db_stats\" WHERE \"dbname\" =~ /^$dbname/ AND $timeFilter GROUP BY dbname, time($__interval) fill(none)", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n dbname,\n avg(((c - c_lag) + (r - r_lag)) / extract(epoch from time - time_lag)) as \"TPS\"\nfrom (\n select \n (data->'xact_commit')::int8 as c, lag((data->'xact_commit')::int8) over w as c_lag,\n (data->'xact_rollback')::int8 as r, lag((data->'xact_rollback')::int8) over w as r_lag,\n time, lag(time) over w as time_lag, dbname\n from db_stats\n where dbname in ($dbname) and $__timeFilter(time)\n window w as (partition by dbname order by time) \n) x\nwhere c >= c_lag and r >= r_lag and time > time_lag\ngroup by 1, 2\norder by 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TPS", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "load_5": "#BA43A9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "Based on pg_stat_statements.calls", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 17, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_dbname", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "stat_statements", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n dbname,\n avg((c - c_lag) / extract(epoch from time - time_lag)) as \"QPS\"\nfrom (\n select \n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n time, lag(time) over w as time_lag, dbname\n from stat_statements_calls\n where dbname in ($dbname) and $__timeFilter(time)\n window w as (partition by dbname order by time) \n) x\nwhere c >= c_lag and time > time_lag\ngroup by 1, 2\norder by 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Queries per Second", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "load_5": "#BA43A9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Requires \"pg_stat_statements\" on server. For regular queries (calls >10)", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 15, + "interval": "15m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_dbname", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "stat_statements", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select non_negative_derivative(last(\"total_time\")) / non_negative_derivative(last(\"calls\")) from \"stat_statements\" WHERE \"dbname\" =~ /^$dbname/ AND $timeFilter AND calls > 100 GROUP BY dbname, time($__interval) fill(none)", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n dbname,\n avg((tt-tt_lag)::numeric / (c-c_lag)) as avg_query_runtime\nfrom (\n select \n (data->>'total_time')::float8 as tt, lag((data->>'total_time')::float8) over w as tt_lag,\n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n time, dbname\n from stat_statements_calls\n where dbname in ($dbname) and $__timeFilter(time)\n window w as (partition by dbname order by time)\n) x\nwhere c > c_lag and tt >= tt_lag\ngroup by 1, 2\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Avg. query runtime", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "TX rollback ratio": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 8, + "interval": "5m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_dbname", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT (non_negative_derivative(mean(\"xact_rollback\")) / (non_negative_derivative(mean(\"xact_commit\")) + non_negative_derivative(mean(\"xact_rollback\"))) * 100) FROM \"db_stats\" WHERE $timeFilter GROUP BY dbname, time($__interval) fill(none)", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n dbname,\n avg(case when c = c_lag and r = r_lag then 0 else (((r - r_lag)::numeric * 100) / ((c - c_lag) + (r - r_lag)) ) end) as tx_err_ratio\nfrom (\n select \n (data->'xact_commit')::int8 as c, lag((data->'xact_commit')::int8) over w as c_lag,\n (data->'xact_rollback')::int8 as r, lag((data->'xact_rollback')::int8) over w as r_lag,\n time, dbname\n from db_stats\n where dbname in ($dbname) and $__timeFilter(time)\n window w as (partition by dbname order by time) \n) x\nwhere c >= c_lag and r >= r_lag\ngroup by 1, 2\norder by 1, 2", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TX error ratio", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "load_5": "#BA43A9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "hiddenSeries": false, + "id": 19, + "interval": "30m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_dbname", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "cpu_load", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT last(\"size_b\") FROM \"db_size\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY dbname, time($__interval) fill(none)", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval) as time,\n dbname,\n avg((data->'size_b')::int8) db_size\nfrom\n db_size\nwhere \n dbname in ($dbname)\n and $__timeFilter(time)\ngroup by\n 1, 2\norder by\n 1, 2", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load_5min" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DB size", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "load_5": "#BA43A9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "Requires according \"metric fetching helpers\" on the monitored server", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "id": 12, + "interval": "15m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "avg_query_runtime", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_dbname", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "cpu_load", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n dbname,\n avg((data->'load_5min')::float8)\nFROM\n cpu_load\nWHERE\n $__timeFilter(time)\n AND dbname in ($dbname)\nGROUP BY 1, 2\nORDER BY 1, 2\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load_5min" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU load", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "#Backends": "#F9E2D2", + "Deadlocks (1h rate)": "#BF1B00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 49 + }, + "hiddenSeries": false, + "id": 9, + "interval": "15m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_dbname", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n dbname,\n sum(tb-tb_lag) as temp_bytes\nfrom (\n select \n (data->>'temp_bytes')::int8 as tb, lag((data->>'temp_bytes')::int8) over w as tb_lag,\n time, dbname\n from db_stats\n where dbname in ($dbname) and $__timeFilter(time)\n window w as (partition by dbname order by time)\n) x\nwhere tb >= tb_lag\ngroup by 1, 2\norder by 1, 2\n", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Temp bytes written", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "bytes", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "DB Size": "#2F575E", + "WAL rate": "#0A50A1" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "fillGradient": 0, + "grid": {}, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "height": "", + "hiddenSeries": false, + "id": 10, + "interval": "15m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_dbname", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "wal", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n dbname,\n avg((wal-wal_lag) / extract(epoch from (time - time_lag))) as \"WAL rate\"\nfrom (\n select \n (data->>'xlog_location_b')::int8 as wal, lag((data->>'xlog_location_b')::int8) over w as wal_lag,\n time, lag(time) over w as time_lag, dbname\n from wal\n where dbname in ($dbname) and $__timeFilter(time)\n window w as (partition by dbname order by time)\n) x\nwhere wal >= wal_lag and time > time_lag\ngroup by 1, 2\norder by 1, 2", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "xlog_location_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1h" + ], + "type": "derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "WAL rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "load_5": "#BA43A9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 0, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 64 + }, + "hiddenSeries": false, + "id": 16, + "interval": "15m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_dbname", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "cpu_load", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT ( non_negative_derivative(mean(\"blks_hit\")) / (non_negative_derivative(mean(\"blks_hit\")) + non_negative_derivative(mean(\"blks_read\")))) * 100 FROM \"db_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY dbname, time($__interval) fill(none)", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n dbname,\n avg( case when hit=hit_lag and read=read_lag then null else (hit-hit_lag)::numeric / ((hit-hit_lag) + (read-read_lag))*100 end) as \"Shared buffers hit ratio\"\nfrom (\n select \n (data->>'blks_hit')::int8 as hit, lag((data->>'blks_hit')::int8) over w as hit_lag,\n (data->>'blks_read')::int8 as read, lag((data->>'blks_read')::int8) over w as read_lag,\n time, dbname\n from db_stats\n where dbname in ($dbname) and $__timeFilter(time)\n window w as (partition by dbname order by time)\n) x\nwhere hit >= hit_lag or read >= read_lag\ngroup by 1, 2\norder by 1, 2", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load_5min" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Shared buffer hit ratio", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "percent", + "label": null, + "logBase": 1, + "max": "105", + "min": null, + "show": true + }, + { + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "Based on pg_stat_activity info", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 73 + }, + "hiddenSeries": false, + "id": 11, + "interval": "15m", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_dbname", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "measurement": "backends", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"idleintransaction\") + mean(\"waiting\") + mean(\"active\") FROM \"backends\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY dbname, time($__interval) fill(none)", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n dbname,\n avg((data->'idleintransaction')::int + (data->'waiting')::int + (data->'active')::int) as sessions\nFROM\n backends\nWHERE\n $__timeFilter(time)\n and dbname in ($dbname)\nGROUP BY\n 1, 2\nORDER BY\n 1, 2\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "idle" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Non-idle sessions", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Brought to you by: \"Cybertec", + "datasource": null, + "editable": true, + "error": false, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 81 + }, + "id": 14, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 22, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "tags": [], + "text": "yes", + "value": "yes" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "current_primaries_only", + "options": [ + { + "selected": true, + "text": "yes", + "value": "yes" + }, + { + "selected": false, + "text": "no", + "value": "no" + } + ], + "query": "yes,no", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": null, + "definition": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics a WHERE case when '$current_primaries_only' = 'yes' then (select (data->'in_recovery_int')::int as in_recovery_int from db_stats where dbname = a.dbname and time > now() - '1h'::interval order by time desc limit 1) = 0 else true end ORDER BY 1;", + "hide": 0, + "includeAll": true, + "index": -1, + "label": null, + "multi": true, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics a WHERE case when '$current_primaries_only' = 'yes' then (select (data->'in_recovery_int')::int as in_recovery_int from db_stats where dbname = a.dbname and time > now() - '1h'::interval order by time desc limit 1) = 0 else true end ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "15m", + "value": "15m" + }, + "hide": 0, + "label": null, + "name": "agg_interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": true, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1m,5m,10m,15m,30m,1h,6h,12h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Global DB overview", + "uid": "global-db-overview", + "variables": { + "list": [] + }, + "version": 1 + } + aws-cloudwatch.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "aws", + "decimals": 1, + "description": "The percentage of CPU utilization.\n\nUnits: Percent", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "height": "150", + "id": 1, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "CPU %", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "dsType": null, + "expression": "", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "highResolution": false, + "id": "", + "metricName": "CPUUtilization", + "namespace": "AWS/RDS", + "orderByTime": "ASC", + "period": "600", + "policy": "default", + "query": "SELECT non_negative_derivative(mean(\"xact_commit\"), 1s) + non_negative_derivative(mean(\"xact_rollback\"), 1s) FROM \"db_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY time($interval) fill(none)", + "rawQuery": true, + "refId": "A", + "region": "$region", + "resultFormat": "time_series", + "returnData": false, + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "statistics": [ + "Average" + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": "6h", + "title": "CPU load %", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "aws", + "decimals": 0, + "description": "The average number of bytes written to disk per second.", + "editable": true, + "error": false, + "format": "bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "height": "150", + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "Write bytes/s", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "dsType": null, + "expression": "", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "highResolution": false, + "id": "", + "metricName": "WriteThroughput", + "namespace": "AWS/RDS", + "orderByTime": "ASC", + "period": "600", + "policy": "default", + "query": "SELECT non_negative_derivative(mean(\"calls\"), 1s) FROM \"stat_statements_calls\" WHERE $timeFilter AND \"dbname\" =~ /^$dbname$/ GROUP BY time(1s) fill(none)", + "rawQuery": true, + "refId": "A", + "region": "$region", + "resultFormat": "time_series", + "returnData": false, + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "statistics": [ + "Average" + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": "6h", + "title": "Write IO", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "aws", + "decimals": 1, + "description": "The average number of bytes read from disk per second.", + "editable": true, + "error": false, + "format": "Bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "height": "150", + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "Read bytes/s", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "dsType": null, + "expression": "", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "highResolution": false, + "id": "", + "metricName": "ReadThroughput", + "namespace": "AWS/RDS", + "orderByTime": "ASC", + "period": "600", + "policy": "default", + "query": "SELECT last(\"size_b\") - first(\"size_b\") FROM \"db_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY time(1h) fill(none)", + "rawQuery": true, + "refId": "A", + "region": "$region", + "resultFormat": "time_series", + "returnData": false, + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "statistics": [ + "Average" + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": "6h", + "timeShift": null, + "title": "Read IO", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "aws", + "description": "", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 0 + }, + "height": "150", + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "Connections", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "dsType": null, + "expression": "", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "highResolution": false, + "id": "", + "metricName": "DatabaseConnections", + "namespace": "AWS/RDS", + "orderByTime": "ASC", + "period": "600", + "policy": "default", + "query": "SELECT non_negative_derivative(last(\"total_time\"), 1h) / non_negative_derivative(last(\"calls\"), 1h) FROM \"stat_statements\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY time($interval) fill(none)", + "rawQuery": true, + "refId": "A", + "region": "$region", + "resultFormat": "time_series", + "returnData": false, + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "statistics": [ + "Average" + ], + "tags": [] + } + ], + "thresholds": "", + "timeFrom": "6h", + "timeShift": null, + "title": "Connections", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "aws", + "decimals": 0, + "description": "The amount of available random access memory.", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 0 + }, + "height": "150", + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "Free mem", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "dsType": null, + "expression": "", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "highResolution": false, + "id": "", + "measurement": "table_bloat_approx_summary", + "metricName": "FreeableMemory", + "namespace": "AWS/RDS", + "orderByTime": "ASC", + "period": "600", + "policy": "default", + "query": "SELECT \"load_5\" FROM \"db_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY time(1h) fill(none)", + "rawQuery": false, + "refId": "A", + "region": "$region", + "resultFormat": "time_series", + "returnData": false, + "select": [ + [ + { + "params": [ + "approx_free_space_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "statistics": [ + "Average" + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ] + } + ], + "thresholds": "", + "timeFrom": "6h", + "timeShift": null, + "title": "Free RAM", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "aws", + "decimals": 1, + "description": "The amount of available storage space.", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "height": "150", + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "alias": "Free storage", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "dsType": null, + "expression": "", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "highResolution": false, + "id": "", + "measurement": "cpu_load", + "metricName": "FreeStorageSpace", + "namespace": "AWS/RDS", + "orderByTime": "ASC", + "period": "1800", + "policy": "default", + "query": "SELECT \"load_5\" FROM \"db_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY time(1h) fill(none)", + "rawQuery": false, + "refId": "A", + "region": "$region", + "resultFormat": "time_series", + "returnData": false, + "select": [ + [ + { + "params": [ + "load_5min" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "statistics": [ + "Average" + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ] + } + ], + "thresholds": "", + "timeFrom": "6h", + "timeShift": null, + "title": "Free disk", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "aws", + "decimals": 1, + "description": "Credit balance/usage apply only to T2 instances", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "db_stats.mean", + "yaxis": 2 + }, + { + "alias": "DB size", + "yaxis": 2 + }, + { + "alias": "blk_read_time", + "yaxis": 2 + }, + { + "alias": "blk_write_time", + "yaxis": 2 + }, + { + "alias": "CPU credit usage", + "yaxis": 2 + }, + { + "alias": "CPU credit balance", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "CPU load", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "expression": "", + "highResolution": false, + "id": "", + "metricName": "CPUUtilization", + "namespace": "AWS/RDS", + "period": "300", + "refId": "B", + "region": "$region", + "returnData": false, + "statistics": [ + "Average" + ] + }, + { + "alias": "CPU credit balance", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "expression": "", + "highResolution": false, + "id": "", + "metricName": "CPUCreditBalance", + "namespace": "AWS/RDS", + "period": "300", + "refId": "A", + "region": "$region", + "returnData": false, + "statistics": [ + "Average" + ] + }, + { + "alias": "CPU credit usage", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "expression": "", + "highResolution": false, + "id": "", + "metricName": "CPUCreditUsage", + "namespace": "AWS/RDS", + "period": "300", + "refId": "C", + "region": "$region", + "returnData": false, + "statistics": [ + "Average" + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU details", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "IO queue depth": "#bf1b00", + "TX rollback ratio": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "aws", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "IO queue depth", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Write ops/s", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "expression": "", + "highResolution": false, + "id": "", + "metricName": "WriteIOPS", + "namespace": "AWS/RDS", + "period": "300", + "refId": "A", + "region": "$region", + "returnData": false, + "statistics": [ + "Average" + ] + }, + { + "alias": "Read ops/s", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "expression": "", + "highResolution": false, + "id": "", + "metricName": "ReadIOPS", + "namespace": "AWS/RDS", + "period": "300", + "refId": "B", + "region": "$region", + "returnData": false, + "statistics": [ + "Average" + ] + }, + { + "alias": "IO queue depth", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "expression": "", + "highResolution": false, + "id": "", + "metricName": "DiskQueueDepth", + "namespace": "AWS/RDS", + "period": "300", + "refId": "C", + "region": "$region", + "returnData": false, + "statistics": [ + "Average" + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IOPS details", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 1, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "#Backends": "#F9E2D2", + "Deadlocks (1h rate)": "#BF1B00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "aws", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 9, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Deadlocks (1h rate)", + "yaxis": 1 + }, + { + "alias": "Temp bytes written", + "yaxis": 2 + }, + { + "alias": "Free RAM", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Free disk space", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "expression": "", + "highResolution": false, + "id": "", + "metricName": "FreeStorageSpace", + "namespace": "AWS/RDS", + "period": "300", + "refId": "B", + "region": "$region", + "returnData": false, + "statistics": [ + "Average" + ] + }, + { + "alias": "WAL size", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "expression": "", + "highResolution": false, + "id": "", + "metricName": "TransactionLogsDiskUsage", + "namespace": "AWS/RDS", + "period": "300", + "refId": "C", + "region": "$region", + "returnData": false, + "statistics": [ + "Average" + ] + }, + { + "alias": "Free RAM", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "expression": "", + "highResolution": false, + "id": "", + "metricName": "FreeableMemory", + "namespace": "AWS/RDS", + "period": "300", + "refId": "A", + "region": "$region", + "returnData": false, + "statistics": [ + "Average" + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk/RAM details", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "DB Size": "#2F575E", + "WAL rate": "#0A50A1" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "aws", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 11 + }, + "height": "", + "id": 10, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "WAL rate (bytes/h)", + "yaxis": 2 + }, + { + "alias": "DB Size", + "yaxis": 2 + }, + { + "alias": "WAL bytes/s", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Write bytes/s", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "dsType": null, + "expression": "", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "highResolution": false, + "id": "", + "metricName": "WriteThroughput", + "namespace": "AWS/RDS", + "orderByTime": "ASC", + "period": "300", + "policy": "default", + "refId": "A", + "region": "$region", + "resultFormat": "time_series", + "returnData": false, + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "statistics": [ + "Average" + ], + "tags": [] + }, + { + "alias": "Read bytes/s", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "dsType": null, + "expression": "", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "highResolution": false, + "id": "", + "metricName": "ReadThroughput", + "namespace": "AWS/RDS", + "orderByTime": "ASC", + "period": "300", + "policy": "default", + "refId": "B", + "region": "$region", + "resultFormat": "time_series", + "returnData": false, + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "statistics": [ + "Average" + ], + "tags": [] + }, + { + "alias": "WAL bytes/s", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "dsType": null, + "expression": "", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "highResolution": false, + "id": "", + "metricName": "TransactionLogsGeneration", + "namespace": "AWS/RDS", + "orderByTime": "ASC", + "period": "300", + "policy": "default", + "refId": "C", + "region": "$region", + "resultFormat": "time_series", + "returnData": false, + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "statistics": [ + "Average" + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IO throughput", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "aws", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Connections", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "expression": "", + "highResolution": false, + "id": "", + "metricName": "DatabaseConnections", + "namespace": "AWS/RDS", + "period": "300", + "refId": "B", + "region": "$region", + "returnData": false, + "statistics": [ + "Average" + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "aws", + "description": "For replicas only!", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Replication lag", + "dimensions": { + "DBInstanceIdentifier": "$dbname" + }, + "expression": "", + "highResolution": false, + "id": "", + "metricName": "ReplicaLag", + "namespace": "AWS/RDS", + "period": "300", + "refId": "B", + "region": "$region", + "returnData": false, + "statistics": [ + "Average" + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Replication lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Brought to you by: \"Cybertec", + "editable": true, + "error": false, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 14, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": "aws", + "definition": "", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "region", + "options": [], + "query": "regions()", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": "aws", + "definition": "", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "dimension_values($region,AWS/RDS,CPUUtilization,DBInstanceIdentifier)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "AWS CloudWatch overview", + "uid": "aws-cloudwatch-overview", + "version": 1 + } + biggest-relations.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Depends on the \"Heatmap\" plugin. Based on latest \"table/index_stats\" data", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "iteration": 1703497578925, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "postgres" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "separator": "." + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "links": [], + "maxDataPoints": 100, + "options": { + "sizeField": "total_relation_size_b", + "textField": "table_name", + "tiling": "treemapSquarify" + }, + "targets": [ + { + "alias": "$tag_table_name", + "datasource": { + "type": "postgres" + }, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time,\n (data->'total_relation_size_b')::int8 as total_relation_size_b,\n tag_data->>'table_full_name'::text as table_name\nFROM\n table_stats\nWHERE\n time = (select max(time) from table_stats WHERE dbname = '$dbname' AND $__timeFilter(time))\n AND dbname = '$dbname'\nORDER BY\n 2 DESC NULLS LAST\nLIMIT 100", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Biggest by total relation size", + "type": "marcusolsson-treemap-panel" + }, + { + "datasource": { + "type": "postgres" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "separator": "." + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 1, + "links": [], + "maxDataPoints": 100, + "options": { + "sizeField": "table_size_b", + "textField": "table_name", + "tiling": "treemapSquarify" + }, + "targets": [ + { + "alias": "$tag_table_name", + "datasource": { + "type": "postgres" + }, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "1h" + ], + "type": "time" + }, + { + "params": [ + "table_full_name" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n (data->'table_size_b')::int8 + coalesce((data->'toast_size_b')::int8, 0) as table_size_b,\n tag_data->>'table_full_name' as table_name\nFROM\n table_stats\nWHERE\n time = (select max(time) from table_stats WHERE dbname = '$dbname' AND $__timeFilter(time))\n AND dbname = '$dbname'\nORDER BY\n 2 DESC NULLS LAST\nLIMIT 100", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_relation_size_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=", + "value": "$dbname" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Biggest tables (w/o indexes)", + "type": "marcusolsson-treemap-panel" + }, + { + "datasource": { + "type": "postgres" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "separator": "." + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 2, + "links": [], + "maxDataPoints": 100, + "options": { + "sizeField": "index_size_b", + "textField": "index_full_name", + "tiling": "treemapSquarify" + }, + "targets": [ + { + "alias": "$tag_index_name", + "datasource": { + "type": "postgres" + }, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n (data->'index_size_b')::int8 as index_size_b,\n tag_data->>'index_full_name' as index_full_name\nFROM\n index_stats\nWHERE\n time = (select max(time) from index_stats WHERE dbname = '$dbname' AND $__timeFilter(time))\n AND dbname = '$dbname'\nORDER BY\n 2 DESC NULLS LAST\nLIMIT 100", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Biggest indexes", + "type": "marcusolsson-treemap-panel" + }, + { + "datasource": { + "type": "postgres" + }, + "editable": true, + "error": false, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 4, + "links": [], + "options": { + "content": "Brought to you by: \"Cybertec", + "mode": "html" + }, + "pluginVersion": "8.5.20", + "transparent": true, + "type": "text" + } + ], + "refresh": "1m", + "schemaVersion": 36, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "postgres" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'table_stats' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Biggest relations treemap", + "uid": "biggest-relations", + "version": 1, + "weekStart": "" + } + change-events.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": null, + "enable": true, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "changes summary", + "query": "select details from object_changes where $timeFilter AND dbname = '$dbname'", + "rawQuery": "SELECT\n time,\n data->>'details' as text\nFROM\n object_changes\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n", + "showIn": 0, + "type": "alert" + } + ] + }, + "description": "use change_events metric for capturing table/index/sproc/postgresql.conf changes", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": { + "Rollback ratio": "#BF1B00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "If there are no vertical red stripes then no changes were detected for that time period", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "rollback ratio", + "yaxis": 2 + }, + { + "alias": "Rollback ratio", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "CPU load", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "cpu_load", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $__interval),\n avg((data->'load_5min')::float8) as \"load_5min\"\nFROM\n cpu_load\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1\nORDER BY 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [] + ], + "table": "cpu_load", + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "\"time\"", + "timeColumnType": "timestamptz", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "alias": "Rollback ratio", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "auto" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $__interval),\n avg( case when (xc = xc_lag and xr = xr_lag) then null else ( ((xr - xr_lag)::numeric*100) / ((xc - xc_lag) + (xr - xr_lag)) ) end ) as \"Rollback ratio\"\nfrom (\n select\n (data->'xact_rollback')::float8 as xr, lag((data->'xact_rollback')::float8) over w as xr_lag,\n (data->'xact_commit')::float8 as xc, lag((data->'xact_rollback')::float8) over w as xc_lag,\n time\n from db_stats\n where dbname = '$dbname'\n and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere xc >= xc_lag\ngroup by 1\norder by 1\n\n\n\n\n\n", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load_5min" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DDL / config change events", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 1, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": null, + "filterNull": false, + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 2, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "dsType": null, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "1s" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "object_changes", + "metricColumn": "none", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n (date_trunc('minute', time))::time as time,\n data->>'event' as event,\n tag_data->>'table' as table\nFROM\n table_changes\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nORDER BY\n time", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "details" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Table changes", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "filterNull": false, + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 3, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "dsType": null, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "1s" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "object_changes", + "metricColumn": "none", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n (date_trunc('minute', time))::time as time,\n data->>'event' as event,\n tag_data->>'index' as index,\n data->>'table' as table\nFROM\n index_changes\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nORDER BY\n time", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "details" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Index changes", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "filterNull": false, + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 4, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "dsType": null, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time::time,\n data->>'event' as event,\n tag_data->>'sproc' as sproc\nFROM\n sproc_changes\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nORDER BY\n time", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Sproc changes", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "postgresql.conf settings changes", + "filterNull": false, + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 5, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "dsType": null, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time::time,\n tag_data->>'setting' as event,\n data->>'value' as value\nFROM\n configuration_changes\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nORDER BY\n time", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Config changes", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Instance level", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "filterNull": false, + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 8, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "dsType": null, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n (date_trunc('minute', time))::time as time,\n tag_data->>'role' as role,\n data->>'event' as event, \n data->>'privilege_type' as privilege_type\nFROM\n privilege_changes\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n AND (data->>'privilege_type' = 'LOGIN'\n OR data->>'privilege_type' = 'SUPERUSER') \nORDER BY\n time, tag_data->>'role', data->>'privilege_type'", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Login and superuser role changes", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Per database", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "filterNull": false, + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 9, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "dsType": null, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n (date_trunc('minute', time))::time as time,\n tag_data->>'role' as role,\n data->>'event' as event, \n data->>'privilege_type' as privilege_type,\n tag_data->>'object' as object\nFROM\n privilege_changes\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n AND data->>'privilege_type' != 'LOGIN'\n AND data->>'privilege_type' != 'SUPERUSER'\nORDER BY\n time, tag_data->>'role', data->>'privilege_type', data->>'object'", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Privilege changes", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 7, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n (select time from settings where $__timeFilter(time) order by time desc limit 1) as \"Time\",\n key,\n value::text as value\nfrom (\n select * from jsonb_each((select data from settings where $__timeFilter(time) order by time desc limit 1))\n) x;", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Settings", + "transform": "table", + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 18, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": null, + "definition": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics ORDER BY 1;", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Change events", + "uid": "change-events", + "version": 1 + } + checkpointer-bgwriter-stats.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": { + "checkpoints_req": "semi-dark-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 3, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "checkpoints_timed", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1h" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "bgwriter", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, '$agg_interval'),\n sum(nullif(ct - ct_lag, 0)) as \"checkpoints_timed\",\n sum(nullif(cr - cr_lag, 0)) as \"checkpoints_req\"\nFROM (\n SELECT\n time,\n (data->'checkpoints_timed')::int8 as ct, lag((data->'checkpoints_timed')::int8) over w as ct_lag,\n (data->'checkpoints_req')::int8 as cr, lag((data->'checkpoints_req')::int8) over w as cr_lag\n FROM\n bgwriter\n WHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n WINDOW w as (order by time)\n) x\nWHERE ct >= ct_lag and cr >= cr_lag\nGROUP BY 1\nORDER BY 1\n", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "checkpoints_timed" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "1h" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checkpoints", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "checkpoint_write_time", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "1h" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "bgwriter", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, '$agg_interval'),\n sum(wt-wt_lag) as \"checkpoint_write_time\",\n sum(st-st_lag) as \"checkpoint_sync_time\"\nFROM (\n\n SELECT\n time,\n (data->'checkpoint_write_time')::int8 as wt, lag((data->'checkpoint_write_time')::int8) over w as wt_lag,\n (data->'checkpoint_sync_time')::int8 as st, lag((data->'checkpoint_sync_time')::int8) over w as st_lag\n FROM\n bgwriter\n WHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n WINDOW w as (order by time)\n) x\nWHERE wt >= wt_lag or st >= st_lag\nGROUP BY 1\nORDER BY 1\n", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "checkpoint_write_time" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1h" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checkpointer Write / Sync durations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 5, + "interval": "60m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "buffers_checkpoint", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "5m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "bgwriter", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n 8192 * sum(buffers_checkpoint-buffers_checkpoint_lag) as \"buffers_checkpoint\",\n 8192 * sum(buffers_clean-buffers_clean_lag) as \"buffers_clean\",\n 8192 * sum(buffers_backend-buffers_backend_lag) as \"buffers_backend\"\nFROM (\n SELECT\n time,\n (data->'buffers_checkpoint')::int8 as buffers_checkpoint, lag((data->'buffers_checkpoint')::int8) over w as buffers_checkpoint_lag,\n (data->'buffers_clean')::int8 as buffers_clean, lag((data->'buffers_clean')::int8) over w as buffers_clean_lag,\n (data->'buffers_backend')::int8 as buffers_backend, lag((data->'buffers_backend')::int8) over w as buffers_backend_lag\n FROM\n bgwriter\n WHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n WINDOW w as (order by time)\n) x\nWHERE buffers_checkpoint >= buffers_checkpoint_lag and buffers_clean >= buffers_clean_lag and buffers_backend >= buffers_backend_lag\nGROUP BY 1\nORDER BY 1\n", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_checkpoint" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1h" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Bgwriter Stats", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Based on pg_stat_database. Requires \"track_io_timing=on\"", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 6, + "interval": "15m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "blk_read_time (backend)", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "5m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n sum(rt-rt_lag) as \"blk_read_time\",\n sum(wt-wt_lag) as \"blk_write_time\"\nFROM (\n SELECT\n time,\n (data->'blk_read_time')::float8 as rt, lag((data->'blk_read_time')::float8) over w as rt_lag,\n (data->'blk_write_time')::float8 as wt, lag((data->'blk_write_time')::float8) over w as wt_lag\n FROM\n db_stats\n WHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n WINDOW w as (order by time)\n) x\nWHERE rt >= rt_lag and wt >= wt_lag\nGROUP BY 1\nORDER BY 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "blk_read_time" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1h" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Backend Read / Write times", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Reads could still be served by the Linux file system cache so they don't necessarily have to come from disk", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 7, + "interval": "", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "heap_blks_read", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "15m" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_io_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n 8192 * sum(heap-heap_lag) as \"heap_blks_read\",\n 8192 * sum(idx-idx_lag) as \"idx_blks_read\",\n 8192 * sum(tidx-tidx_lag) as \"tidx_blks_read\",\n 8192 * sum(toast-toast_lag) as \"toast_blks_read\"\nFROM (\n SELECT\n time,\n (data->'heap_blks_read')::float8 as heap, lag((data->'heap_blks_read')::float8) over w as heap_lag,\n (data->'idx_blks_read')::float8 as idx, lag((data->'idx_blks_read')::float8) over w as idx_lag,\n (data->'tidx_blks_read')::float8 as tidx, lag((data->'tidx_blks_read')::float8) over w as tidx_lag,\n (data->'toast_blks_read')::float8 as toast, lag((data->'toast_blks_read')::float8) over w as toast_lag\n FROM\n table_io_stats\n WHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n WINDOW w as (partition by tag_data->>'table_full_name' order by time)\n) x\nWHERE heap >= heap_lag and idx >= idx_lag\nGROUP BY 1\nORDER BY 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "heap_blks_read" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1h" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Table / Index / Toast Blocks Read", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Brought to you by: \"Cybertec", + "datasource": null, + "editable": true, + "error": false, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 9, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 22, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'bgwriter' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "tags": [], + "text": "1h", + "value": "1h" + }, + "hide": 0, + "label": null, + "name": "agg_interval", + "options": [ + { + "selected": false, + "text": "1s", + "value": "1s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": true, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "3h", + "value": "3h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1s,1m,5m,10m,15m,30m,1h,3h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Checkpointer / Bgwriter / Block IO Stats", + "uid": "checkpointer-bgwriter-stats", + "version": 1 + } + documentation.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "content": "# Available metric tables overview\n\n### Metrics that are actually gathered need to be configured for every DB separately - for that open the Web UI config page [here](http://0.0.0.0:8080/dbs) or modify the \"pgwatch.monitored_host\" table directly in the \"pgwatch\" database\n\n* **archiver** - WAL archiving status/progress\n* **backends** - active, total, waiting sessions\n* **bgwriter** - pg_stat_bgwriter snapshots\n* **blocking_locks** - detailed info on sessions that are waiting\n* **change_events** - summary on object (DDL) and server configuration changes\n* **cpu_load** - CPU load info acquired via a plpython sproc (/pgwatch/sql/metric_fetching_helpers/)\n* **db_stats** - pg_stat_database snapshots\n* **db_size** - DB size info\n* **index_stats** - pg_stat_user_indexes snapshots\n* **kpi** - most important high level metrics\n* **locks** - different locktype (page, tuple, ...) counts. for usable data one should set the polling interval very low\n* **locks_mode** - different lock-mode (exclusive, share) counts. for usable data one should set the polling interval very low\n* **replication** - pg_stat_replication info (including replica lag)\n* **replication_slots** - pg_replication_slots info\n* **sproc_stats** - pg_stat_user_functions snapshots\n* **table_io_stats** - pg_statio_user_tables snapshots\n* **pgbouncer_stats** - pgbouncer (1.8+) statistics\n* **pgpool_stats** - pgpool-II (3.0+) statistics\n* **recommendations** - suggestion on possible security / performance issues\n* **stat_activity_realtime** - realtime view on pg_stat_activity\n* **stat_statements** - pg_stat_statements snapshots (requires the extension)\n* **stat_statements_calls** - total query count according to pg_stat_statements\n* **table_bloat_approx_summary** - bloat summary for the whole DB (needs *pgstattuple* extension)\n* **table_stats** - pg_stat_user_tables snapshots\n* **wal** - pg_current_(xlog_location|wal_lsn) values\n* **wal_receiver** - pg_stat_wal_receiver data on replica side\n* **wal_size** - WAL folder size\n\n# For getting started with Grafana in general start [here](http://docs.grafana.org/guides/getting_started/)\n\n\n# When stuck then additional support and consultations are available from Cybertec [here](https://www.cybertec-postgresql.com/en/contact/)", + "datasource": null, + "editable": true, + "error": false, + "gridPos": { + "h": 21, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "links": [], + "mode": "markdown", + "title": "Documentation", + "type": "text" + }, + { + "content": "Brought to you by: \"Cybertec", + "editable": true, + "error": false, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 2, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Documentation", + "uid": "documentation", + "version": 1 + } + lock-details.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "Based on pg_locks", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_lockmode", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "lockmode" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "locks_mode", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $__interval),\n lockmode,\n avg(count)\nFROM (\nSELECT\n time,\n tag_data->>'lockmode' as lockmode,\n (data->'count')::int8 as count\nFROM\n locks_mode\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n) x\nGROUP BY\n 1, 2\nORDER BY\n 1, 2\n", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "count" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Locking details (exp. scale)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 10, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric IN ('locks', 'locks_mode') ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Lock details", + "uid": "lock-details", + "version": 1 + } + pgbouncer-stats.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "#614d93", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": 1, + "description": "Current \"Transaction Per Second\" based on total_xact_count", + "editable": true, + "error": false, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "height": "100", + "hideTimeOverride": true, + "id": 1, + "interval": "2m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "alias": "TPS", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "pgbouncer_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $__interval),\n avg((c - c_lag) / extract(epoch from time - time_lag)) as tps\nfrom (\n select \n (data->>'total_xact_count')::int8 as c, lag((data->>'total_xact_count')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from pgbouncer_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_xact_count" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "15m", + "title": "TPS", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "#629e51", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": 1, + "description": "Current \"Queries Per Second\" based on total_query_count", + "editable": true, + "error": false, + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "height": "100", + "hideTimeOverride": true, + "id": 2, + "interval": "2m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "alias": "QPS", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "pgbouncer_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $__interval),\n avg((c - c_lag) / extract(epoch from time - time_lag)) as tps\nfrom (\n select \n (data->>'total_query_count')::int8 as c, lag((data->>'total_query_count')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from pgbouncer_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_query_count" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "15m", + "title": "QPS", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": 1, + "description": "Current value based on derivative of total_query_time / total_query_count", + "editable": true, + "error": false, + "format": "µs", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "height": "100", + "hideTimeOverride": true, + "id": 3, + "interval": "2m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "alias": "avg_query_runtime", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_derivative(last(\"total_query_time\"), 1h) / non_negative_derivative(last(\"total_query_count\"), 1h) FROM \"pgbouncer_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "rawSql": "select\n$__timeGroup(time, $__interval),\navg((tt-tt_lag)::numeric / (c-c_lag))\n\nfrom (\n\n select \n (data->>'total_query_time')::float8 as tt, lag((data->>'total_query_time')::float8) over w as tt_lag,\n (data->>'total_query_count')::int8 as c, lag((data->>'total_query_count')::int8) over w as c_lag,\n time\n from pgbouncer_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere c > c_lag and tt >= tt_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "15m", + "timeShift": null, + "title": "Avg. query runtime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "#890f02", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": 1, + "description": "Current avg. wait time to get a connection, per query", + "editable": true, + "error": false, + "format": "µs", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 0 + }, + "height": "100", + "hideTimeOverride": true, + "id": 4, + "interval": "2m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "alias": "db_size_change_last_hour", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "pgbouncer_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_derivative(last(\"total_wait_time\")) / non_negative_derivative(last(\"total_query_count\")) FROM \"pgbouncer_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "rawSql": "select\n$__timeGroup(time, $__interval),\navg((tt-tt_lag)::numeric / (c-c_lag))\n\nfrom (\n\n select \n (data->>'total_wait_time')::float8 as tt, lag((data->>'total_wait_time')::float8) over w as tt_lag,\n (data->>'total_query_count')::int8 as c, lag((data->>'total_query_count')::int8) over w as c_lag,\n time\n from pgbouncer_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere c > c_lag and tt >= tt_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_wait_time" + ], + "type": "field" + }, + { + "params": [], + "type": "last" + }, + { + "params": [ + "1m" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "1h", + "timeShift": null, + "title": "Pool wait time", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#052b51", + "#64b0c8", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": 1, + "description": "Current Incoming traffic based on total_received.", + "editable": true, + "error": false, + "format": "Bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 0 + }, + "height": "100", + "hideTimeOverride": true, + "id": 5, + "interval": "2m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "alias": "bytes_received", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "pgbouncer_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $__interval),\n avg((c - c_lag) / extract(epoch from time - time_lag)) as tps\nfrom (\n select \n (data->>'total_received')::int8 as c, lag((data->>'total_received')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from pgbouncer_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_received" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "15m", + "timeShift": null, + "title": "Incoming traffic", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "#e24d42", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": null, + "decimals": 1, + "description": "Based on total_sent", + "editable": true, + "error": false, + "format": "Bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "height": "100", + "hideTimeOverride": true, + "id": 6, + "interval": "2m", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " ", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "pgbouncer_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $__interval),\n avg((c - c_lag) / extract(epoch from time - time_lag)) as tps\nfrom (\n select \n (data->>'total_sent')::int8 as c, lag((data->>'total_sent')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from pgbouncer_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_sent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "", + "timeFrom": "15m", + "timeShift": null, + "title": "Outgoing traffic", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": { + "TPS": "#584477" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "decimals": null, + "description": "Based on total_query_count", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 7, + "interval": "2m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "TPS", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "pgbouncer_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $__interval),\n avg((c - c_lag) / extract(epoch from time - time_lag)) as \"TPS\"\nfrom (\n select \n (data->>'total_xact_count')::int8 as c, lag((data->>'total_xact_count')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from pgbouncer_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_xact_count" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Pool wait time": "#890f02", + "TX rollback ratio": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "decimals": null, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 8, + "interval": "2m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Pool wait time", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "db_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_derivative(mean(\"total_wait_time\")) / non_negative_derivative(mean(\"total_query_count\")) FROM \"pgbouncer_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "rawSql": "select\n$__timeGroup(time, $__interval),\navg((tt-tt_lag)::numeric / (c-c_lag)) as \"Pool wait time\"\n\nfrom (\n\n select \n (data->>'total_wait_time')::float8 as tt, lag((data->>'total_wait_time')::float8) over w as tt_lag,\n (data->>'total_query_count')::int8 as c, lag((data->>'total_query_count')::int8) over w as c_lag,\n time\n from pgbouncer_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere c > c_lag and tt >= tt_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "blks_hit" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pool wait time per Query", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "µs", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "#Backends": "#F9E2D2", + "Deadlocks (1h rate)": "#BF1B00", + "QPS": "#508642" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "decimals": null, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 9, + "interval": "2m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Deadlocks (1h rate)", + "yaxis": 1 + }, + { + "alias": "Temp bytes written", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "QPS", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "pgbouncer_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $__interval),\n avg((c - c_lag) / extract(epoch from time - time_lag)) as \"QPS\"\nfrom (\n select \n (data->>'total_query_count')::int8 as c, lag((data->>'total_query_count')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from pgbouncer_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_query_count" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "QPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "DB Size": "#2F575E", + "Incoming byterate": "#65c5db", + "Incoming rate": "#6ed0e0", + "WAL rate": "#0A50A1" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "decimals": null, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 9 + }, + "height": "", + "id": 10, + "interval": "2m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Incoming rate", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "pgbouncer_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $__interval),\n avg((c - c_lag) / extract(epoch from time - time_lag)) as \"Incoming rate\"\nfrom (\n select \n (data->>'total_received')::int8 as c, lag((data->>'total_received')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from pgbouncer_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_received" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Incoming traffic rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Avg. runtime": "#ef843c" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "decimals": null, + "description": "", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 11, + "interval": "2m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Avg. runtime", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_derivative(mean(\"total_query_time\"), 1h) / non_negative_derivative(mean(\"total_query_count\"), 1h) FROM \"pgbouncer_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "rawSql": "select\n$__timeGroup(time, $__interval),\navg((tt-tt_lag)::numeric / (c-c_lag)) as \"Avg. runtime\"\n\nfrom (\n\n select \n (data->>'total_query_time')::float8 as tt, lag((data->>'total_query_time')::float8) over w as tt_lag,\n (data->>'total_query_count')::int8 as c, lag((data->>'total_query_count')::int8) over w as c_lag,\n time\n from pgbouncer_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time)\n) x\nwhere c > c_lag and tt >= tt_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Avg. query runtime", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "µs", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Outgoing byterate": "#e24d42", + "Outgoing rate": "#e24d42", + "load_5": "#BA43A9" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 12, + "interval": "2m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Outgoing rate", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "pgbouncer_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $__interval),\n avg((c - c_lag) / extract(epoch from time - time_lag)) as \"Outgoing rate\"\nfrom (\n select \n (data->>'total_sent')::int8 as c, lag((data->>'total_sent')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from pgbouncer_stats\n where dbname = '$dbname' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_sent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing traffic rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Brought to you by: \"Cybertec", + "editable": true, + "error": false, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 14, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": false, + "schemaVersion": 18, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'pgbouncer_stats' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "PgBouncer stats", + "uid": "pgbouncer-stats", + "version": 1 + } + pgpool-stats.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "columns": [], + "datasource": null, + "description": "Based on \"SHOW POOL_NODES\" and \"SHOW POOL_PROCESSES\".", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "pageSize": null, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "", + "align": "auto", + "colorMode": "row", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "mappingType": 1, + "pattern": "status_num", + "thresholds": [ + "1", + "1" + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "replication_delay", + "thresholds": [], + "type": "number", + "unit": "bytes" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n dbname,\n tag_data->>'node_id' as node_id,\n data->>'role' as role,\n data->>'status' as status,\n case when data->>'status' = 'up' then 1 else 0 end as status_num,\n data->>'hostname' as hostname,\n data->>'port' as port,\n data->>'lb_weight' as lb_weight,\n data->>'load_balance_node' as load_balance_node,\n data->>'replication_delay' as replication_delay,\n data->>'select_cnt' as select_cnt,\n data->>'processes_total' as processes_total,\n data->>'processes_active' as processes_active, \n data->>'last_status_change' as last_status_change, \n date_trunc('second', time::timestamp)::time as time\nfrom\n pgpool_stats\nwhere\n time = (select max(time) from pgpool_stats where dbname in ($dbname) and $__timeFilter(time) group by dbname)\n and dbname in ($dbname)\norder by\n dbname,\n node_id,\n time desc\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Pool node status", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n (max((data->'select_cnt')::int8) - min((data->'select_cnt')::int8))::int8 as select_cnt,\n dbname || ' (node: ' || (tag_data->>'node_id')::text || ')' as node_id\nFROM\n pgpool_stats\nWHERE\n $__timeFilter(time)\n AND dbname in ($dbname)\nGROUP BY\n 1, 3\nORDER BY\n 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "SELECT load balancing per node ($agg_interval avg.)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Brought to you by: \"Cybertec", + "datasource": null, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 4, + "mode": "html", + "timeFrom": null, + "timeShift": null, + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 22, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": null, + "definition": "select distinct dbname from admin.all_distinct_dbname_metrics where metric = 'pgpool_stats';", + "hide": 0, + "includeAll": true, + "index": -1, + "label": null, + "multi": true, + "name": "dbname", + "options": [], + "query": "select distinct dbname from admin.all_distinct_dbname_metrics where metric = 'pgpool_stats';", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "10m", + "value": "10m" + }, + "hide": 0, + "label": null, + "name": "agg_interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "2m", + "value": "2m" + }, + { + "selected": false, + "text": "3m", + "value": "3m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": true, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1m,2m,3m,5m,10m,15m,30m,1h,6h,12h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Pgpool Stats", + "uid": "pgpool-stats", + "variables": { + "list": [] + }, + "version": 1 + } + show-plans-realtime.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Based on the pg_show_plans extension", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "columns": [], + "datasource": null, + "description": "Based on the \"pg_show_plans\" extension and \"pg_stat_activity\" infos", + "fontSize": "100%", + "gridPos": { + "h": 22, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "/duration/", + "thresholds": [], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "plan", + "preserveFormat": true, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "query", + "preserveFormat": true, + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n (data->'max_s')::int AS max_duration,\n (data->'avg_s')::int AS avg_duration,\n (data->'count')::int AS count,\n data->>'users' AS users,\n data->>'query' AS query,\n data->>'plan' AS plan\nFROM\n show_plans_realtime\nWHERE\n time = (select max(time) from show_plans_realtime where $__timeFilter(time) and dbname = '$dbname' group by dbname)\n AND dbname = '$dbname'\nORDER BY\n max_duration DESC\nLIMIT\n 25\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Realtime execution plans", + "transform": "table", + "type": "table" + }, + { + "content": "Brought to you by: \"Cybertec", + "datasource": null, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 4, + "mode": "html", + "options": {}, + "timeFrom": null, + "timeShift": null, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "5s", + "schemaVersion": 20, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "datasource": null, + "definition": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'show_plans_realtime' ORDER BY 1;", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'show_plans_realtime' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Realtime execution plans", + "uid": "realtime-execution-plans", + "version": 1 + } + single-query-details.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "datasource": null, + "description": "For one query", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "avg_runtime", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "stat_statements", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_derivative(mean(\"total_time\")) / non_negative_derivative(mean(\"calls\")) FROM \"stat_statements\" WHERE \"queryid\" =~ /^$queryid$/ AND $timeFilter GROUP BY time($__interval), \"queryid\" fill(null)", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(case when c = c_lag then null else (tt - tt_lag) / (c - c_lag) end) as avg_runtime\nfrom (\n select \n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n (data->>'total_time')::float8 as tt, lag((data->>'total_time')::float8) over w as tt_lag,\n time\n from stat_statements\n where dbname = '$dbname' and tag_data @> '{\"queryid\": \"$queryid\"}' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_time" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1h" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "queryid", + "operator": "=~", + "value": "/^$queryid$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Avg. runtime ($agg_interval avg.)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 2, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "calls", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "queryid" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "stat_statements", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n sum(c - c_lag) as \"calls\"\nfrom (\n select \n (data->>'calls')::int8 as c, lag((data->>'calls')::int8) over w as c_lag,\n time, lag(time) over w as time_lag\n from stat_statements\n where dbname = '$dbname' and tag_data @> '{\"queryid\": \"$queryid\"}' and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere c >= c_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "calls" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "$rate_unit" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "queryid", + "operator": "=~", + "value": "/^$queryid$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Calls ($agg_interval aggregate)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 1, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "total_runtime", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "queryid" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "stat_statements", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n sum(case when tt = tt_lag then null else tt - tt_lag end) as \"total_time\"\nfrom (\n select\n time,\n (data->>'total_time')::float8 as tt, lag((data->>'total_time')::float8) over w as tt_lag\n from stat_statements\n where dbname = '$dbname' and tag_data @> '{\"queryid\": \"$queryid\"}'and $__timeFilter(time)\n window w as (order by time) \n) x\nwhere tt >= tt_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_time" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "$rate_unit" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "queryid", + "operator": "=~", + "value": "/^$queryid$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total runtime ($agg_interval aggregate)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 4, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "sb_hit_ratio", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "stat_statements", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_derivative(mean(\"shared_blks_hit\")) / (non_negative_derivative(mean(\"shared_blks_hit\")) + non_negative_derivative(mean(\"shared_blks_read\"))) * 100 FROM \"stat_statements\" WHERE \"queryid\" =~ /^$queryid$/ AND $timeFilter GROUP BY time($__interval), \"queryid\" fill(null)", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(case when hit = hit_lag and read = read_lag then null else ((hit-hit_lag)::numeric * 100) / ((hit-hit_lag) + (read-read_lag)) end) as \"sb_hit_ratio\"\nfrom (\n select \n (data->'shared_blks_hit')::int8 as hit, lag((data->'shared_blks_hit')::int8) over w as hit_lag,\n (data->'shared_blks_read')::int8 as read, lag((data->'shared_blks_read')::int8) over w as read_lag,\n time\n from stat_statements\n where dbname = '$dbname' and $__timeFilter(time) and tag_data @> '{\"queryid\": \"$queryid\"}'\n window w as (order by time)\n) x\nwhere hit >= hit_lag and read >= read_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "shared_blks_hit" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1h" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "queryid", + "operator": "=~", + "value": "/^$queryid$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Shared Buffers Hit Ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 5, + "interval": "5m", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "temp_blks_written", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "queryid" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "stat_statements", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n sum(read-read_lag) as \"temp_blks_read\",\n sum(written-written_lag) as \"temp_blks_written\"\nfrom (\n select \n (data->'temp_blks_read')::int8 as read, lag((data->'temp_blks_read')::int8) over w as read_lag,\n (data->'temp_blks_written')::int8 as written, lag((data->'temp_blks_written')::int8) over w as written_lag,\n time\n from stat_statements\n where dbname = '$dbname' and $__timeFilter(time) and tag_data @> '{\"queryid\": \"$queryid\"}'\n window w as (order by time)\n) x\nwhere read >= read_lag and written >= written_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "temp_blks_written" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "$rate_unit" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "queryid", + "operator": "=~", + "value": "/^$queryid$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Temp Blocks Read/Written ($agg_interval aggregate)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "FYI - blk_write_time can easily be 0 as normally changes happen only in the shared buffers and are written out by the checkpointer / background writer.", + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 6, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "blk_read_time", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "queryid" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "stat_statements", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n sum(read-read_lag) as \"blk_read_time\",\n sum(write-write_lag) as \"blk_write_time\"\nfrom (\n select \n (data->'blk_read_time')::int8 as read, lag((data->'blk_read_time')::int8) over w as read_lag,\n (data->'blk_write_time')::int8 as write, lag((data->'blk_write_time')::int8) over w as write_lag,\n time\n from stat_statements\n where dbname = '$dbname' and $__timeFilter(time) and tag_data @> '{\"queryid\": \"$queryid\"}'\n window w as (order by time)\n) x\nwhere read >= read_lag and write>= write_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "blk_read_time" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "$rate_unit" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "queryid", + "operator": "=~", + "value": "/^$queryid$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Backend block Read/Write time ($agg_interval aggregate)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 8, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "io_time_pct", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT ((non_negative_derivative(mean(\"blk_read_time\")) + non_negative_derivative(mean(\"blk_write_time\"))) / non_negative_derivative(mean(\"total_time\"))) * 100 FROM \"stat_statements\" WHERE \"queryid\" =~ /^$queryid$/ AND $timeFilter GROUP BY time($__interval), \"queryid\" fill(null)", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg(case when tt = tt_lag then null else (read-read_lag+write-write_lag)::numeric*100 / (tt-tt_lag) end) as \"io_time_pct\"\nfrom (\n select \n (data->'blk_read_time')::int8 as read, lag((data->'blk_read_time')::int8) over w as read_lag,\n (data->'blk_write_time')::int8 as write, lag((data->'blk_write_time')::int8) over w as write_lag,\n (data->'total_time')::int8 as tt, lag((data->'total_time')::int8) over w as tt_lag,\n time\n from stat_statements\n where dbname = '$dbname' and $__timeFilter(time) and tag_data @> '{\"queryid\": \"$queryid\"}'\n window w as (order by time)\n) x\nwhere read >= read_lag and write>= write_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "% of total_time spent in direct IO ($agg_interval avg.)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": null, + "fontSize": "100%", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 9, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "Metric", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value", + "sanitize": true, + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "dsType": null, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n tag_data->>'query' as \"Value\"\nfrom stat_statements\nwhere dbname = '$dbname' and tag_data @> '{\"queryid\": \"$queryid\"}'\norder by time desc\nlimit 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "SQL", + "transform": "table", + "type": "table" + }, + { + "content": "Brought to you by: \"Cybertec", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 7, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": false, + "schemaVersion": 18, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'stat_statements' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": null, + "definition": "select distinct tag_data->>'queryid' from stat_statements WHERE time > current_date -3 and dbname = '$dbname' order by 1;", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "queryid", + "options": [], + "query": "select distinct tag_data->>'queryid' from stat_statements WHERE time > current_date -3 and dbname = '$dbname' order by 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "tags": [], + "text": "30m", + "value": "30m" + }, + "hide": 0, + "label": null, + "name": "agg_interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": true, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1m,5m,10m,15m,30m,1h,6h,12h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Single query details", + "uid": "single-query-details", + "version": 1 + } + sproc-details.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": "2m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "calls", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "sproc_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n time,\n sum(calls - calls_lag) as calls\nFROM (\nSELECT\n $__timeGroup(time, $agg_interval),\n (data->'sp_calls')::int8 as calls,\n lag((data->'sp_calls')::int8) OVER(order by time) as calls_lag\nFROM\n sproc_stats\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n AND tag_data @> '{\"function_full_name\": \"$sproc_name\"}'\n) x\nWHERE calls >= calls_lag\nGROUP BY 1\nORDER BY 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "sp_calls" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1h" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + }, + { + "condition": "AND", + "key": "function_full_name", + "operator": "=~", + "value": "/^$sproc_name$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Calls ($agg_interval aggregate)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 3, + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 2, + "interval": "2m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "total_time", + "dsType": null, + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "auto" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "sproc_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg( case when calls = calls_lag then null else (total_time - total_time_lag) / (calls - calls_lag) end) as total_time,\n avg( case when calls = calls_lag then null else (self_time - self_time_lag) / (calls - calls_lag) end) as self_time\nFROM (\n SELECT\n (data->'sp_calls')::int8 as calls, lag((data->'sp_calls')::int8) over w as calls_lag,\n (data->'self_time')::float as self_time, lag((data->'self_time')::float) over w as self_time_lag,\n (data->'total_time')::float as total_time, lag((data->'total_time')::float) over w as total_time_lag,\n time, lag(time) over w as time_lag\n FROM\n sproc_stats\n WHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n AND tag_data @> jsonb_build_object('function_full_name', '$sproc_name')\n WINDOW w AS (order by time)\n) x\nWHERE time > time_lag AND calls >= calls_lag\nGROUP BY 1\nORDER BY 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_time" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1h" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + }, + { + "condition": "AND", + "key": "function_full_name", + "operator": "=~", + "value": "/^$sproc_name$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Avg. runtime ($agg_interval avg.)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 3, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 19, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'sproc_stats' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": null, + "definition": "select distinct tag_data->>'function_full_name' from sproc_stats where time > current_date-7 and dbname = '$dbname' and tag_data ? 'function_full_name' order by 1;", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "sproc_name", + "options": [], + "query": "select distinct tag_data->>'function_full_name' from sproc_stats where time > current_date-7 and dbname = '$dbname' and tag_data ? 'function_full_name' order by 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "tags": [], + "text": "10m", + "value": "10m" + }, + "hide": 0, + "label": null, + "name": "agg_interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": true, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1m,5m,10m,15m,30m,1h,6h,12h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Sproc details", + "uid": "sproc-details", + "version": 1 + } + sprocs-top.json: |- + { + "annotations": { + "list": [ + { + "$hashKey": null, + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "columns": [], + "datasource": null, + "description": "As per \"sproc_stats\" measurement. Needs track_functions parameter set to 'pl' on monitored DBs.", + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [], + "options": {}, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "$hashKey": null, + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "$hashKey": null, + "alias": "total_runtime", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "$hashKey": null, + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Sproc details' dashboard for that sproc", + "linkUrl": "/d/sproc-details?var-dbname=$dbname&var-sproc_name=$__cell&from=$__from&to=$__to", + "pattern": "function_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "calls", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "oid", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "$hashKey": null, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "sproc_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT top(\"spread\", \"function_full_name\", \"oid\", $top) FROM (SELECT spread(\"total_time\") FROM \"sproc_stats\" WHERE $timeFilter AND \"dbname\" = '$dbname' GROUP BY time(3650d), \"function_full_name\", \"oid\" fill(none) ) WHERE spread > 0", + "rawQuery": true, + "rawSql": "SELECT\n max((data->'total_time')::float8) - min((data->'total_time')::float8) as top,\n max((data->'sp_calls')::int8) - min((data->'sp_calls')::int8) as calls,\n tag_data->>'function_full_name' as function_full_name,\n tag_data->>'oid' as oid\nFROM\n sproc_stats\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 3, 4\nORDER BY 1 DESC NULLS LAST\nLIMIT $top", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "total_time" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "By Total Runtime", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "As per \"sproc_stats\" measurement. Needs track_functions parameter set to 'pl' on monitored DBs.", + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 5, + "links": [], + "options": {}, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "$hashKey": null, + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "$hashKey": null, + "alias": "total_runtime", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "$hashKey": null, + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Sproc details' dashboard for that sproc", + "linkUrl": "/d/sproc-details?var-dbname=$dbname&var-sproc_name=$__cell&from=$__from&to=$__to", + "pattern": "function_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "calls", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "oid", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "$hashKey": null, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "sproc_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT top(\"spread\", \"function_full_name\", \"oid\", $top) FROM (SELECT spread(\"self_time\") FROM \"sproc_stats\" WHERE $timeFilter AND \"dbname\" = '$dbname' GROUP BY time(3650d), \"function_full_name\", \"oid\" fill(none) ) WHERE spread > 0", + "rawQuery": true, + "rawSql": "SELECT\n max((data->'self_time')::float8) - min((data->'self_time')::float8) as top,\n max((data->'sp_calls')::int8) - min((data->'sp_calls')::int8) as calls, \n tag_data->>'function_full_name' as function_full_name,\n tag_data->>'oid' as oid\nFROM\n sproc_stats\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 3, 4\nORDER BY 1 DESC NULLS LAST\nLIMIT $top", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "total_time" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "By Total Self Runtime", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "As per \"sproc_stats\" measurement. Needs track_functions parameter set to 'pl' on monitored DBs.", + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 4, + "links": [], + "options": {}, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "$hashKey": null, + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "$hashKey": null, + "alias": "calls", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "$hashKey": null, + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Sproc details' dashboard for that sproc", + "linkUrl": "/d/sproc-details?var-dbname=$dbname&var-sproc_name=$__cell&from=$__from&to=$__to", + "pattern": "function_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "total_time", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "oid", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "$hashKey": null, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "sproc_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT top(\"spread\", \"function_full_name\", \"oid\", $top) FROM (SELECT spread(\"sp_calls\") FROM \"sproc_stats\" WHERE $timeFilter AND \"dbname\" = '$dbname' GROUP BY time(3650d), \"function_full_name\", \"oid\" fill(none) ) WHERE spread > 0", + "rawQuery": true, + "rawSql": "SELECT\n max((data->'sp_calls')::int8) - min((data->'sp_calls')::int8) as top,\n max((data->'total_time')::float8) - min((data->'total_time')::float8) as total_time,\n tag_data->>'function_full_name' as function_full_name,\n tag_data->>'oid' as oid\nFROM\n sproc_stats\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 3, 4\nORDER BY 1 DESC NULLS LAST\nLIMIT $top", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "total_time" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "By Calls", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "As per \"sproc_stats\" measurement. Needs track_functions parameter set to 'pl' on monitored DBs.", + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 3, + "links": [], + "options": {}, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "$hashKey": null, + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "$hashKey": null, + "alias": "avg_runtime", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "$hashKey": null, + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Sproc details' dashboard for that sproc", + "linkUrl": "/d/sproc-details?var-dbname=$dbname&var-sproc_name=$__cell&from=$__from&to=$__to", + "pattern": "function_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "calls", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "oid", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "$hashKey": null, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "sproc_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select top(avg, \"function_full_name\", \"oid\", $top) AS top from (select mean(t) / mean(c) AS avg from (SELECT spread(\"total_time\") AS t, spread(\"sp_calls\") AS c FROM \"sproc_stats\" WHERE $timeFilter AND dbname = '$dbname' GROUP BY time(3650d), \"function_full_name\", \"oid\" fill(none) ) where $timeFilter AND c > 0 group by time(3650d), \"function_full_name\", \"oid\" )", + "rawQuery": true, + "rawSql": "SELECT\n total_time / calls as top,\n calls,\n function_full_name,\n oid\nFROM (\n SELECT\n tag_data->>'function_full_name' as function_full_name,\n tag_data->>'oid' as oid,\n max((data->'total_time')::float8) - min((data->'total_time')::float8) as total_time,\n max((data->'sp_calls')::int8) - min((data->'sp_calls')::int8) as calls\n FROM\n sproc_stats\n WHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n GROUP BY 1, 2\n) x\nWHERE calls > 0\nORDER BY 1 DESC NULLS LAST\nLIMIT $top", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "total_time" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "By Avg. Runtime", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "As per \"sproc_stats\" measurement. Needs track_functions parameter set to 'pl' on monitored DBs.", + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 6, + "links": [], + "options": {}, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "$hashKey": null, + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "$hashKey": null, + "alias": "avg_runtime", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "$hashKey": null, + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Sproc details' dashboard for that sproc", + "linkUrl": "/d/sproc-details?var-dbname=$dbname&var-sproc_name=$__cell&from=$__from&to=$__to", + "pattern": "function_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "calls", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "oid", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "$hashKey": null, + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "sproc_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select top(avg, \"function_full_name\", \"oid\", $top) AS top from (select mean(t) / mean(c) AS avg from (SELECT spread(\"self_time\") AS t, spread(\"sp_calls\") AS c FROM \"sproc_stats\" WHERE $timeFilter AND dbname = '$dbname' GROUP BY time(3650d), \"function_full_name\", \"oid\" fill(none) ) where $timeFilter AND c > 0 group by time(3650d), \"function_full_name\", \"oid\" )", + "rawQuery": true, + "rawSql": "SELECT\n total_time / calls as top,\n calls,\n function_full_name,\n oid\nFROM (\n SELECT\n tag_data->>'function_full_name' as function_full_name,\n tag_data->>'oid' as oid,\n max((data->'self_time')::float8) - min((data->'self_time')::float8) as total_time,\n max((data->'sp_calls')::int8) - min((data->'sp_calls')::int8) as calls\n FROM\n sproc_stats\n WHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n GROUP BY 1, 2\n) x\nWHERE calls > 0\nORDER BY 1 DESC NULLS LAST\nLIMIT $top", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "total_time" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "By Avg. Self Runtime", + "transform": "table", + "type": "table" + }, + { + "content": "Brought to you by: \"Cybertec", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 8, + "links": [], + "mode": "html", + "options": {}, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": false, + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": null, + "text": null + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'sproc_stats' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "tags": [], + "text": "5", + "value": "5" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "top", + "options": [ + { + "$hashKey": null, + "selected": false, + "text": "1", + "value": "1" + }, + { + "hashKey": null, + "selected": false, + "text": "3", + "value": "3" + }, + { + "$hashKey": null, + "selected": true, + "text": "5", + "value": "5" + }, + { + "$hashKey": null, + "selected": false, + "text": "10", + "value": "10" + }, + { + "$hashKey": null, + "selected": false, + "text": "15", + "value": "15" + }, + { + "$hashKey": null, + "selected": false, + "text": "20", + "value": "20" + }, + { + "$hashKey": null, + "selected": false, + "text": "30", + "value": "30" + }, + { + "$hashKey": null, + "selected": false, + "text": "40", + "value": "40" + }, + { + "$hashKey": null, + "selected": false, + "text": "50", + "value": "50" + } + ], + "query": "1,3,5,10,15,20,30,40,50", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Top Sprocs", + "uid": "top-sprocs", + "version": 1 + } + stat-activity-realtime.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "columns": [], + "datasource": null, + "description": "Top 25 longest running queries from pg_stat_activity. Query texts are compacted", + "fontSize": "100%", + "gridPos": { + "h": 24, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": {}, + "pageSize": null, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "pid", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": "value", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "duration", + "thresholds": [ + "60", + "300" + ], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": "value", + "colors": [ + "rgba(50, 172, 45, 0.97)", + "#C4162A", + "rgba(245, 54, 54, 0.9)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "waiting", + "thresholds": [ + "1", + "2" + ], + "type": "string", + "unit": "short", + "valueMaps": [ + { + "text": "no", + "value": "0" + }, + { + "text": "yes", + "value": "1" + } + ] + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "blocking_pids", + "thresholds": [], + "type": "string", + "unit": "short", + "valueMaps": [] + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n tag_data->>'pid' AS pid,\n data->>'user' AS user,\n data->>'appname' AS appname,\n data->>'ip' AS ip,\n (data->'duration_s')::int AS duration,\n data->>'waiting' AS waiting,\n coalesce(data->>'blocking_pids', '') AS blocking_pids,\n data->>'query' AS query\nFROM\n stat_activity_realtime\nWHERE\n dbname = '$dbname'\n AND time = (SELECT max(time) FROM stat_activity_realtime WHERE dbname = '$dbname' AND $__timeFilter(time))\n AND (data->'duration_s')::int > $min_duration_s\nORDER BY\n duration DESC\nLIMIT 25\n\n\n\n\n\n\n\n\n\n\n\n\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Stat Activity", + "transform": "table", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 21, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "datasource": null, + "definition": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'stat_activity_realtime' ORDER BY 1;", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'stat_activity_realtime' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "text": "1", + "value": "1" + }, + "hide": 0, + "label": null, + "name": "min_duration_s", + "options": [ + { + "text": "1", + "value": "1" + } + ], + "query": "1", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "hidden": false, + "refresh_intervals": [ + "1s", + "2s", + "3s", + "4s", + "5s", + "10s", + "30s", + "1m", + "5m", + "15m" + ] + }, + "timezone": "", + "title": "Stat Activity Realtime", + "uid": "stat-activity-realtime", + "version": 1 + } + stat-activity.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1697, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "firstNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": null, + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n sum((data->>'count')::integer) as \" \",\n data->>'query' as \"query\"\nfrom stat_activity\nwhere dbname = '$dbname' and $__timeFilter(time)\n and case when '$hide_pgwatch_generated' = 'yes' then\n not data->>'query' ~* E'/\\\\*\\\\W*pgwatch_generated\\\\W*\\\\*/'\n else\n true\n end\ngroup by 1,3\norder by 1;\n\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Active sessions detail", + "type": "timeseries" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "count" + }, + "properties": [ + { + "id": "custom.width", + "value": 61 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 6, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.1.4", + "targets": [ + { + "datasource": null, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select sum((data->>'count')::integer) as count, data->>'query' as query\nfrom stat_activity\nwhere dbname = '$dbname'\n and $__timeFilter(time)\n and case when '$hide_pgwatch_generated' = 'yes' then\n not data->>'query' ~* E'/\\\\*\\\\W*pgwatch_generated\\\\W*\\\\*/'\n else\n true\n end\ngroup by 2\norder by 1 desc;", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Queries", + "type": "table" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "current": { + "text": null, + "value": null + }, + "datasource": null, + "definition": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'stat_activity' ORDER BY 1;", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'stat_activity' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "1m", + "value": "1m" + }, + "hide": 0, + "name": "agg_interval", + "options": [ + { + "selected": true, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1m,5m,10m,15m,30m,1h,6h,12h,1d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "selected": false, + "text": "yes", + "value": "yes" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "hide_pgwatch_generated", + "options": [ + { + "selected": true, + "text": "yes", + "value": "yes" + }, + { + "selected": false, + "text": "no", + "value": "no" + } + ], + "query": "yes,no", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Stat Activity", + "uid": "stat-activity", + "version": 11, + "weekStart": "" + } + stat-statements-sql-search.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "columns": [], + "description": "", + "fontSize": "100%", + "gridPos": { + "h": 16, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": {}, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/time/", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "calls", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Go to Query Details dash", + "linkUrl": "/d/single-query-details?var-dbname=$dbname&var-queryid=$__cell", + "mappingType": 1, + "pattern": "/queryid/", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "/last_exec/", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n total_time, mean_time, calls as total_calls, seconds_since_last_exec, queryid, query\nFROM (\n\nSELECT\n DISTINCT ON (queryid) queryid, time, total_time, mean_time, calls, query\nFROM (\n\nSELECT\n time, (data->'total_time')::float as total_time, (data->'total_time')::float / (data->'calls')::int8 as mean_time, (data->'calls')::int8 as calls, tag_data->>'queryid' as queryid, tag_data->>'query' as query\nFROM stat_statements\nWHERE\n dbname = '$dbname'\n AND time > now() - '$calls_interval'::interval\n AND tag_data->>'query' ~* '$sql_fragment'\n AND CASE WHEN length('$sql_fragment') > 2 THEN true ELSE false END\n) x\nORDER BY queryid, time DESC, mean_time, calls, query\n\n) y\nJOIN LATERAL \n (select (extract(epoch from now() - prev.time))::int8 as seconds_since_last_exec\n from stat_statements prev\n where dbname = '$dbname' and time > now() - '$calls_interval'::interval\n and prev.time < y.time and (prev.data->'calls')::int8 != y.calls\n order by time desc limit 1\n) z on true\n\nORDER BY total_time DESC LIMIT 100", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Matching Queries", + "transform": "table", + "type": "table" + }, + { + "content": "Brought to you by: \"Cybertec", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 4, + "mode": "html", + "options": {}, + "timeFrom": null, + "timeShift": null, + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 19, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": null, + "definition": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'stat_statements' ORDER BY 1;", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'stat_statements' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "text": "", + "value": "" + }, + "hide": 0, + "label": "SQL search fragment (case insensitive, 3 char min.)", + "name": "sql_fragment", + "options": [ + { + "text": "", + "value": "" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "12h", + "value": "12h" + }, + "hide": 0, + "label": "Having some executions within last", + "name": "calls_interval", + "options": [ + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": true, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": { + "hidden": false, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Stat Statements SQL Search", + "uid": "stat-statements-sql-search", + "version": 1 + } + stat-statements-top-fast.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Assumes no pg_stat_statements reset appeared in the selected time frame", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "columns": [], + "datasource": null, + "description": "", + "fontSize": "90%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 16, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "link": false, + "linkTargetBlank": true, + "linkTooltip": "", + "linkUrl": "", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Query ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Single query details' dashboard for that queryid", + "linkUrl": "/d/single-query-details?orgId=1&var-dbname=$dbname&var-queryid=$__cell&from=$__from&to=$__to", + "pattern": "queryid", + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Total runtime", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "alias": "Query", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": true, + "linkTooltip": "Use 'edit' mode to be able to copy the query text", + "linkUrl": "", + "pattern": "query", + "preserveFormat": false, + "sanitize": true, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "approx_pct_db_total_time", + "thresholds": [], + "type": "number", + "unit": "percent" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Avg. runtime", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "Calls", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "ass", + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT top(\"spread\", \"queryid\", \"query\", $top) FROM (SELECT spread(\"total_time\") FROM \"stat_statements\" WHERE $timeFilter AND \"dbname\" = '$dbname' GROUP BY time(3650d), \"queryid\", \"query\" fill(none) ) WHERE spread > 0", + "rawQuery": true, + "rawSql": "select\n tt as top,\n tt / calls as \"Avg. runtime\",\n calls as \"Calls\",\n 100 * tt / (select sum(tt_agg) from (select tag_data->>'queryid', max((data->'total_time')::float) - min((data->'total_time')::float) as tt_agg from stat_statements where dbname = '$dbname' and $__timeFilter(time) group by 1 having count(*) > 1) x) as approx_pct_db_total_time,\n queryid,\n (select data->>'users' from stat_statements where $__timeFilter(time) and dbname = '$dbname' and tag_data->>'queryid' = queryid order by time desc limit 1) as \"Users\", \n case when length(query) > 250 then query::varchar(250) || '...' else query end as query\nfrom (\n\n select\n tag_data->>'queryid' as queryid, tag_data->>'query' as query,\n max((data->>'total_time')::numeric) - min((data->>'total_time')::numeric) as tt,\n max((data->>'calls')::int8) - min((data->>'calls')::int8) as calls,\n min(time) as min_time,\n max(time) as max_time\n from stat_statements\n where dbname = '$dbname'\n and $__timeFilter(time)\n and case when '$hide_pgwatch_generated' = 'yes' then not tag_data->>'query' ~* E'\\\\(extract\\\\(\\\\$\\\\d+\\\\W*from\\\\W*now\\\\(\\\\)\\\\)\\\\W?\\\\*\\\\W*\\\\$\\\\d+\\\\).*::\\\\w+\\\\W+as\\\\W+epoch_ns\\\\W*,'\n and not tag_data->>'query' ~* E'/\\\\*\\\\W*pgwatch_generated\\\\W*\\\\*/' else true end\n group by\n 1, 2\n\n) x\nwhere calls > 0 and tt > 0\norder by 1 desc nulls last\nlimit $top\n\n", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top queries by total runtime", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "", + "fontSize": "90%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 15, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "link": false, + "linkTargetBlank": true, + "linkTooltip": "", + "linkUrl": "", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Query ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Single query details' dashboard for that queryid", + "linkUrl": "/d/single-query-details?orgId=1&var-dbname=$dbname&var-queryid=$__cell&from=$__from&to=$__to", + "pattern": "queryid", + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Avg. runtime", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "alias": "Query", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": true, + "linkTooltip": "Use 'edit' mode to be able to copy the query text", + "linkUrl": "", + "pattern": "query", + "preserveFormat": false, + "sanitize": true, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "Total time", + "thresholds": [], + "type": "number", + "unit": "ms" + } + ], + "targets": [ + { + "alias": "ass", + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select top(avg, \"queryid\", \"query\", $top) AS top from (select mean(t) / mean(c) AS avg from (SELECT spread(\"total_time\") AS t, spread(\"calls\") AS c FROM \"stat_statements\" WHERE $timeFilter AND dbname = '$dbname' GROUP BY time(3650d), \"queryid\", \"query\" fill(none) ) where $timeFilter AND c > 0 group by time(3650d), \"queryid\", \"query\" )", + "rawQuery": true, + "rawSql": "select\n tt / calls as top,\n tt as \"Total time\",\n calls as \"Calls\",\n queryid,\n (select data->>'users' from stat_statements where $__timeFilter(time) and dbname = '$dbname' and tag_data->>'queryid' = queryid order by time desc limit 1) as \"Users\",\n case when length(query) > 200 then query::varchar(200) || '...' else query end as query\nfrom (\n select\n tag_data->>'queryid' as queryid, tag_data->>'query' as query,\n max((data->>'total_time')::numeric) - min((data->>'total_time')::numeric) as tt,\n max((data->>'calls')::int8) - min((data->>'calls')::int8) as calls\n from stat_statements\n where dbname = '$dbname'\n and $__timeFilter(time)\n and case when '$hide_pgwatch_generated' = 'yes' then not tag_data->>'query' ~* E'\\\\(extract\\\\(\\\\$\\\\d+\\\\W*from\\\\W*now\\\\(\\\\)\\\\)\\\\W?\\\\*\\\\W*\\\\$\\\\d+\\\\).*::\\\\w+\\\\W+as\\\\W+epoch_ns\\\\W*,'\n and not tag_data->>'query' ~* E'/\\\\*\\\\W*pgwatch_generated\\\\W*\\\\*/' else true end\n group by\n 1, 2\n) x\nwhere calls > 0 and tt > 0\norder by 1 desc nulls last\nlimit $top\n\n", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top queries by avg. runtime", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "", + "fontSize": "90%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 14, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "link": false, + "linkTargetBlank": true, + "linkTooltip": "", + "linkUrl": "", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Query ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Single query details' dashboard for that queryid", + "linkUrl": "/d/single-query-details?orgId=1&var-dbname=$dbname&var-queryid=$__cell&from=$__from&to=$__to", + "pattern": "queryid", + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Calls ", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Query", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": true, + "linkTooltip": "Use 'edit' mode to be able to copy the query text", + "linkUrl": "", + "pattern": "query", + "preserveFormat": false, + "sanitize": true, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "Total time", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Avg. runtime", + "thresholds": [], + "type": "number", + "unit": "ms" + } + ], + "targets": [ + { + "alias": "ass", + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT top(\"spread\", \"queryid\", \"query\", $top) FROM (SELECT spread(\"calls\") FROM \"stat_statements\" WHERE $timeFilter AND \"dbname\" = '$dbname' GROUP BY time(3650d), \"queryid\", \"query\" fill(none) ) WHERE spread > 0", + "rawQuery": true, + "rawSql": "select\n calls as top,\n total_time as \"Total time\",\n total_time / calls as \"Avg. runtime\",\n queryid,\n (select data->>'users' from stat_statements where $__timeFilter(time) and dbname = '$dbname' and tag_data->>'queryid' = queryid order by time desc limit 1) as \"Users\", \n case when length(query) > 200 then query::varchar(200) || '...' else query end as query\nfrom (\n select\n tag_data->>'queryid' as queryid, tag_data->>'query' as query,\n max((data->>'calls')::int8) - min((data->>'calls')::int8) as calls,\n max((data->>'total_time')::numeric) - min((data->>'total_time')::numeric) as total_time\n from stat_statements\n where dbname = '$dbname'\n and $__timeFilter(time)\n and case when '$hide_pgwatch_generated' = 'yes' then not tag_data->>'query' ~* E'\\\\(extract\\\\(\\\\$\\\\d+\\\\W*from\\\\W*now\\\\(\\\\)\\\\)\\\\W?\\\\*\\\\W*\\\\$\\\\d+\\\\).*::\\\\w+\\\\W+as\\\\W+epoch_ns\\\\W*,'\n and not tag_data->>'query' ~* E'/\\\\*\\\\W*pgwatch_generated\\\\W*\\\\*/' else true end \n group by\n 1, 2\n) x\nwhere calls > 0\norder by 1 desc nulls last\nlimit $top\n\n", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top queries by calls", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "total_time - blk_read_time -blk_write_time. Requires track_io_timing=on", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 13, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU time", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 1, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "alias": "Query ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Opens 'Single query details' dashboard for that queryid", + "linkUrl": "/d/single-query-details?orgId=1&var-dbname=$dbname&var-queryid=$__cell&from=$__from&to=$__to", + "pattern": "queryid", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Query", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": true, + "linkTooltip": "Use 'edit' mode to be able to copy the query text", + "linkUrl": "", + "pattern": "query", + "sanitize": true, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Avg. runtime", + "thresholds": [], + "type": "number", + "unit": "ms" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT top(\"spread\", \"queryid\", \"query\", $top) FROM (SELECT spread(\"blk_read_time\") + spread(\"blk_write_time\") AS spread FROM \"stat_statements\" WHERE $timeFilter AND \"dbname\" = '$dbname' GROUP BY time(3650d), \"queryid\", \"query\" fill(none) ) WHERE spread > 0", + "rawQuery": true, + "rawSql": "select\n (tt - wt - rt)::int8 as top,\n calls as \"Calls\",\n tt / calls as \"Avg. runtime\",\n queryid,\n (select data->>'users' from stat_statements where $__timeFilter(time) and dbname = '$dbname' and tag_data->>'queryid' = queryid order by time desc limit 1) as \"Users\", \n case when length(query) > 200 then query::varchar(200) || '...' else query end as query\nfrom (\n select\n tag_data->>'queryid' as queryid, tag_data->>'query' as query,\n max((data->>'total_time')::float8) - min((data->>'total_time')::float8) as tt,\n max((data->>'blk_read_time')::float8) - min((data->>'blk_read_time')::float8) as rt,\n max((data->>'blk_write_time')::float8) - min((data->>'blk_write_time')::float8) as wt,\n max((data->>'calls')::int8) - min((data->>'calls')::int8) as calls\n from stat_statements\n where dbname = '$dbname'\n and $__timeFilter(time)\n and case when '$hide_pgwatch_generated' = 'yes' then not tag_data->>'query' ~* E'\\\\(extract\\\\(\\\\$\\\\d+\\\\W*from\\\\W*now\\\\(\\\\)\\\\)\\\\W?\\\\*\\\\W*\\\\$\\\\d+\\\\).*::\\\\w+\\\\W+as\\\\W+epoch_ns\\\\W*,'\n and not tag_data->>'query' ~* E'/\\\\*\\\\W*pgwatch_generated\\\\W*\\\\*/' else true end \n group by\n 1, 2\n) x\nwhere tt - wt - rt > 0 and calls > 0\norder by 1 desc nulls last\nlimit $top\n\n", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top queries by CPU time", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Does not include Shared Buffer activities. Same as 'By total runtime' if no direct IO was performed. Requires track_io_timing=on", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 12, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IO time", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 1, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "alias": "Query ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Opens 'Single query details' dashboard for that queryid", + "linkUrl": "/d/single-query-details?orgId=1&var-dbname=$dbname&var-queryid=$__cell&from=$__from&to=$__to", + "pattern": "queryid", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Query", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": true, + "linkTooltip": "Use 'edit' mode to be able to copy the query text", + "linkUrl": "", + "pattern": "query", + "sanitize": true, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "Total time", + "thresholds": [], + "type": "number", + "unit": "ms" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT top(\"spread\", \"queryid\", \"query\", $top) FROM (SELECT spread(\"blk_read_time\") + spread(\"blk_write_time\") AS spread FROM \"stat_statements\" WHERE $timeFilter AND \"dbname\" = '$dbname' GROUP BY time(3650d), \"queryid\", \"query\" fill(none) ) WHERE spread > 0", + "rawQuery": true, + "rawSql": "select\n wt + rt as top,\n tt as \"Total time\",\n queryid,\n (select data->>'users' from stat_statements where $__timeFilter(time) and dbname = '$dbname' and tag_data->>'queryid' = queryid order by time desc limit 1) as \"Users\", \n case when length(query) > 200 then query::varchar(200) || '...' else query end as query\nfrom (\n select\n tag_data->>'queryid' as queryid, max(tag_data->>'query') as query,\n max((data->>'blk_read_time')::float8) - min((data->>'blk_read_time')::float8) as rt,\n max((data->>'blk_write_time')::float8) - min((data->>'blk_write_time')::float8) as wt,\n max((data->>'total_time')::float8) - min((data->>'total_time')::float8) as tt\n from stat_statements\n where dbname = '$dbname'\n and $__timeFilter(time)\n and case when '$hide_pgwatch_generated' = 'yes' then not tag_data->>'query' ~* E'\\\\(extract\\\\(\\\\$\\\\d+\\\\W*from\\\\W*now\\\\(\\\\)\\\\)\\\\W?\\\\*\\\\W*\\\\$\\\\d+\\\\).*::\\\\w+\\\\W+as\\\\W+epoch_ns\\\\W*,'\n and not tag_data->>'query' ~* E'/\\\\*\\\\W*pgwatch_generated\\\\W*\\\\*/' else true end \n group by\n 1\n) x\nwhere rt + wt > 0.1\norder by 1 desc nulls last\nlimit $top\n\n", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top queries by direct (backend) IO", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Shared buffer + temp buffer reading / writing", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 11, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Block bandwith", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 1, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "decbytes" + }, + { + "alias": "Query ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Single query details' dashboard for that queryid", + "linkUrl": "/d/single-query-details?orgId=1&var-dbname=$dbname&var-queryid=$__cell&from=$__from&to=$__to", + "pattern": "queryid", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Query", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": true, + "linkTooltip": "Use 'edit' mode to be able to copy the query text", + "linkUrl": "", + "pattern": "query", + "sanitize": true, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "Per call", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Avg. runtime", + "thresholds": [], + "type": "number", + "unit": "ms" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT top(\"spread\", \"queryid\", \"query\", $top) FROM (SELECT spread(\"blk_read_time\") + spread(\"blk_write_time\") AS spread FROM \"stat_statements\" WHERE $timeFilter AND \"dbname\" = '$dbname' GROUP BY time(3650d), \"queryid\", \"query\" fill(none) ) WHERE spread > 0", + "rawQuery": true, + "rawSql": "select\n (shared_blks_hit + shared_blks_read + shared_blks_dirtied + shared_blks_written)*8192 /*assuming default 8k block size*/ as top,\n (shared_blks_hit + shared_blks_read + shared_blks_dirtied + shared_blks_written)*8192 / calls as \"Per call\",\n calls as \"Calls\",\n total_time / calls as \"Avg. runtime\",\n queryid,\n (select data->>'users' from stat_statements where $__timeFilter(time) and dbname = '$dbname' and tag_data->>'queryid' = queryid order by time desc limit 1) as \"Users\", \n case when length(query) > 200 then query::varchar(200) || '...' else query end as query\nfrom (\n select\n tag_data->>'queryid' as queryid,\n tag_data->>'query' as query,\n max((data->>'shared_blks_hit')::int8) - min((data->>'shared_blks_hit')::int8) as shared_blks_hit,\n max((data->>'shared_blks_read')::int8) - min((data->>'shared_blks_read')::int8) as shared_blks_read,\n max((data->>'shared_blks_dirtied')::int8) - min((data->>'shared_blks_dirtied')::int8) as shared_blks_dirtied,\n max((data->>'shared_blks_written')::int8) - min((data->>'shared_blks_written')::int8) as shared_blks_written,\n max((data->>'calls')::int8) - min((data->>'calls')::int8) as calls,\n max((data->>'total_time')::float) - min((data->>'total_time')::float) as total_time\n from stat_statements\n where dbname = '$dbname'\n and $__timeFilter(time)\n and case when '$hide_pgwatch_generated' = 'yes' then not tag_data->>'query' ~* E'\\\\(extract\\\\(\\\\$\\\\d+\\\\W*from\\\\W*now\\\\(\\\\)\\\\)\\\\W?\\\\*\\\\W*\\\\$\\\\d+\\\\).*::\\\\w+\\\\W+as\\\\W+epoch_ns\\\\W*,'\n and not tag_data->>'query' ~* E'/\\\\*\\\\W*pgwatch_generated\\\\W*\\\\*/' else true end\n group by\n 1, 2\n) x\nwhere calls > 0\norder by 1 desc nulls last\nlimit $top", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top by block bandwith (assuming 8K blocks)", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Based on pg_stat_statements.temp_blks_written", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 17, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Temp blocks written", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 1, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "decbytes" + }, + { + "alias": "Query ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Single query details' dashboard for that queryid", + "linkUrl": "/d/single-query-details?orgId=1&var-dbname=$dbname&var-queryid=$__cell&from=$__from&to=$__to", + "pattern": "queryid", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Query", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": true, + "linkTooltip": "Use 'edit' mode to be able to copy the query text", + "linkUrl": "", + "pattern": "query", + "sanitize": true, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "Per call", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "/runtime/", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "Temp blocks read", + "thresholds": [], + "type": "number", + "unit": "bytes" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT top(\"spread\", \"queryid\", \"query\", $top) FROM (SELECT spread(\"blk_read_time\") + spread(\"blk_write_time\") AS spread FROM \"stat_statements\" WHERE $timeFilter AND \"dbname\" = '$dbname' GROUP BY time(3650d), \"queryid\", \"query\" fill(none) ) WHERE spread > 0", + "rawQuery": true, + "rawSql": "select\n temp_blks_written * 8192 /*assuming default 8k block size*/ as top,\n (temp_blks_read * 8192) / calls as \"Temp blocks read\",\n total_time as \"Total runtime\",\n calls as \"Calls\",\n total_time / calls as \"Avg. runtime\",\n queryid,\n (select data->>'users' from stat_statements where $__timeFilter(time) and dbname = '$dbname' and tag_data->>'queryid' = queryid order by time desc limit 1) as \"Users\", \n case when length(query) > 200 then query::varchar(200) || '...' else query end as query\nfrom (\n select\n tag_data->>'queryid' as queryid,\n tag_data->>'query' as query,\n max((data->>'temp_blks_read')::int8) - min((data->>'temp_blks_read')::int8) as temp_blks_read,\n max((data->>'temp_blks_written')::int8) - min((data->>'temp_blks_written')::int8) as temp_blks_written,\n max((data->>'calls')::int8) - min((data->>'calls')::int8) as calls,\n max((data->>'total_time')::float) - min((data->>'total_time')::float) as total_time\n from stat_statements\n where dbname = '$dbname'\n and $__timeFilter(time)\n and case when '$hide_pgwatch_generated' = 'yes' then not tag_data->>'query' ~* E'\\\\(extract\\\\(\\\\$\\\\d+\\\\W*from\\\\W*now\\\\(\\\\)\\\\)\\\\W?\\\\*\\\\W*\\\\$\\\\d+\\\\).*::\\\\w+\\\\W+as\\\\W+epoch_ns\\\\W*,'\n and not tag_data->>'query' ~* E'/\\\\*\\\\W*pgwatch_generated\\\\W*\\\\*/' else true end\n group by\n 1, 2\n) x\nwhere temp_blks_written > 0\norder by 1 desc nulls last\nlimit $top", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top by temp blocks (assuming 8K blocks)", + "transform": "table", + "type": "table" + }, + { + "content": "Brought to you by: \"Cybertec", + "datasource": null, + "editable": true, + "error": false, + "gridPos": { + "h": 3, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 4, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 18, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n0 as time,\ncoalesce ((\n\nselect 1 from (\nselect\n (data->'xact_commit')::int8 as xact_commit,\n lag((data->'xact_commit')::int8) over (order by time) as xact_commit_lag\nfrom\n db_stats\nwhere\n $__timeFilter(time)\n and dbname = '$dbname'\n) x\nwhere xact_commit_lag > xact_commit\nlimit 1\n\n), 0) as was_reset", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1,1", + "title": "", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + }, + { + "op": "=", + "text": "", + "value": "0" + }, + { + "op": "=", + "text": "WARNING: global stats reset detected for selected time range, results are unreliable - use the slower \"Stat Statements Top\" dashboard", + "value": "1" + } + ], + "valueName": "avg" + } + ], + "schemaVersion": 22, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": null, + "value": null + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'stat_statements' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "tags": [], + "text": "5", + "value": "5" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "top", + "options": [ + { + "selected": false, + "text": "1", + "value": "1" + }, + { + "selected": false, + "text": "3", + "value": "3" + }, + { + "selected": false, + "text": "5", + "value": "5" + }, + { + "selected": false, + "text": "10", + "value": "10" + }, + { + "selected": false, + "text": "15", + "value": "15" + }, + { + "selected": false, + "text": "20", + "value": "20" + }, + { + "selected": false, + "text": "30", + "value": "30" + }, + { + "selected": false, + "text": "40", + "value": "40" + }, + { + "selected": true, + "text": "50", + "value": "50" + } + ], + "query": "1,3,5,10,15,20,30,40,50", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "current": { + "tags": [], + "text": "yes", + "value": "yes" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "hide_pgwatch_generated", + "options": [ + { + "selected": true, + "text": "yes", + "value": "yes" + }, + { + "selected": false, + "text": "no", + "value": "no" + } + ], + "query": "yes,no", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Stat statements Top (Fast)", + "uid": "stat-statements-top-fast", + "variables": { + "list": [] + }, + "version": 1 + } + stat-statements-top-visual.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": null, + "description": "Assumes that pg_stat_statement data has not been reset in the selected time frame. Click on the graph data links for the 'Query details' view.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ + { + "targetBlank": true, + "title": "Query details", + "url": "/d/single-query-details?$__all_variables&var-queryid=${__series.name}&${__url_time_range}" + } + ] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n max((data->>'calls')::int8) - min((data->>'calls')::int8) as calls,\n tag_data->>'queryid' as queryid\nFROM\n stat_statements\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n AND tag_data->>'queryid' IN (\n select tag_data->>'queryid'\n from stat_statements\n where $__timeFilter(time) and dbname = '$dbname'\n group by tag_data->>'queryid'\n having max((data->>'calls')::int8) - min((data->>'calls')::int8) > 0\n order by max((data->>'calls')::int8) - min((data->>'calls')::int8) desc\n limit $top\n )\nGROUP BY 1, 3\nORDER by 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top $top queries by calls", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": null, + "description": "Assumes that pg_stat_statement data has not been reset in the selected time frame. Click on the graph data links for the 'Query details' view.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 3, + "interval": "", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ + { + "targetBlank": true, + "title": "Query details", + "url": "/d/single-query-details?$__all_variables&var-queryid=${__series.name}&${__url_time_range}" + } + ] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n max((data->>'total_time')::float) - min((data->>'total_time')::float) as total_time,\n tag_data->>'queryid' as queryid\nFROM\n stat_statements\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n AND tag_data->>'queryid' IN (\n select tag_data->>'queryid'\n from stat_statements\n where $__timeFilter(time) and dbname = '$dbname'\n group by tag_data->>'queryid'\n having max((data->>'total_time')::float) - min((data->>'total_time')::float) > 0\n order by max((data->>'total_time')::float) - min((data->>'total_time')::float) desc\n limit $top\n )\nGROUP BY 1, 3\nORDER by 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top $top queries by total time", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Brought to you by: \"Cybertec", + "datasource": null, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 5, + "mode": "html", + "options": {}, + "timeFrom": null, + "timeShift": null, + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 22, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": null, + "value": null + }, + "datasource": null, + "definition": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'stat_statements' ORDER BY 1;", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'stat_statements' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "15m", + "value": "15m" + }, + "hide": 0, + "label": null, + "name": "agg_interval", + "options": [ + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": true, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "5m,10m,15m,30m,1h,6h,12h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "5", + "value": "5" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "top", + "options": [ + { + "selected": false, + "text": "1", + "value": "1" + }, + { + "selected": false, + "text": "3", + "value": "3" + }, + { + "selected": true, + "text": "5", + "value": "5" + }, + { + "selected": false, + "text": "7", + "value": "7" + }, + { + "selected": false, + "text": "10", + "value": "10" + }, + { + "selected": false, + "text": "15", + "value": "15" + } + ], + "query": "1,3,5,7,10,15", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Stat Statements Top (Visual)", + "uid": "stat-statements-top-visual", + "version": 1 + } + stat-statements-top.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "columns": [], + "datasource": null, + "description": "", + "fontSize": "90%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "link": false, + "linkTargetBlank": true, + "linkTooltip": "", + "linkUrl": "", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Query ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Single query details' dashboard for that queryid", + "linkUrl": "/d/single-query-details?orgId=1&var-dbname=$dbname&var-queryid=$__cell&from=$__from&to=$__to", + "pattern": "queryid", + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Total runtime", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "alias": "Query", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": true, + "linkTooltip": "Use 'edit' mode to be able to copy the query text", + "linkUrl": "", + "pattern": "query", + "preserveFormat": false, + "sanitize": true, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "approx_pct_db_total_time", + "thresholds": [], + "type": "number", + "unit": "percent" + } + ], + "targets": [ + { + "alias": "ass", + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n top,\n approx_pct_db_total_time,\n queryid,\n (select data->>'users' from stat_statements where $__timeFilter(time) and dbname = '$dbname' and tag_data->>'queryid' = queryid order by time desc limit 1) as \"Users\",\n query\nfrom (\nselect\n (sum(tt - tt_lag))::int8 as top,\n round(((sum(tt - tt_lag))::int8 / total::numeric)*100, 1) as approx_pct_db_total_time,\n queryid,\n case when length(query) > 250 then query::varchar(250) || '...' else query end as query\nfrom (\nselect\n *,\n sum(tt - tt_lag) over() as total\nfrom (\n select\n tag_data->>'queryid' as queryid, tag_data->>'query' as query, data->>'users' as users,\n (data->>'total_time')::float8 as tt, lag((data->>'total_time')::float8) over w as tt_lag,\n time\n from stat_statements\n where dbname = '$dbname'\n and $__timeFilter(time)\n and case when '$hide_pgwatch_generated' = 'yes' then not tag_data->>'query' ~* E'\\\\(extract\\\\(\\\\$\\\\d+\\\\W*from\\\\W*now\\\\(\\\\)\\\\)\\\\W?\\\\*\\\\W*\\\\$\\\\d+\\\\).*::\\\\w+\\\\W+as\\\\W+epoch_ns\\\\W*,'\n and not tag_data->>'query' ~* E'/\\\\*\\\\W*pgwatch_generated\\\\W*\\\\*/' else true end\n window w as (partition by tag_data->>'queryid' order by time)\n) x\nwhere tt >= tt_lag\n) y\ngroup by total,queryid,query\nhaving sum(tt - tt_lag) > 0\norder by 1 desc nulls last\nlimit $top\n) z\norder by 1 desc nulls last", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top queries by total runtime", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "", + "fontSize": "90%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 6, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "link": false, + "linkTargetBlank": true, + "linkTooltip": "", + "linkUrl": "", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Query ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Single query details' dashboard for that queryid", + "linkUrl": "/d/single-query-details?orgId=1&var-dbname=$dbname&var-queryid=$__cell&from=$__from&to=$__to", + "pattern": "queryid", + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Avg. runtime", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "alias": "Query", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": true, + "linkTooltip": "Use 'edit' mode to be able to copy the query text", + "linkUrl": "", + "pattern": "query", + "preserveFormat": false, + "sanitize": true, + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "alias": "ass", + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n top,\n queryid,\n (select data->>'users' from stat_statements where $__timeFilter(time) and dbname = '$dbname' and tag_data->>'queryid' = queryid order by time desc limit 1) as \"Users\",\n query\nfrom (\nselect\n (avg((tt - tt_lag)::numeric / (calls - calls_lag)))::int8 as top,\n queryid,\n case when length(query) > 200 then query::varchar(200) || '...' else query end as query\nfrom (\n select\n tag_data->>'queryid' as queryid, tag_data->>'query' as query, data->>'users' as users,\n (data->>'total_time')::float8 as tt, lag((data->>'total_time')::float8) over w as tt_lag,\n (data->>'calls')::int8 as calls, lag((data->>'calls')::int8) over w as calls_lag,\n time\n from stat_statements\n where dbname = '$dbname'\n and $__timeFilter(time)\n and case when '$hide_pgwatch_generated' = 'yes' then not tag_data->>'query' ~* E'\\\\(extract\\\\(\\\\$\\\\d+\\\\W*from\\\\W*now\\\\(\\\\)\\\\)\\\\W?\\\\*\\\\W*\\\\$\\\\d+\\\\).*::\\\\w+\\\\W+as\\\\W+epoch_ns\\\\W*,'\n and not tag_data->>'query' ~* E'/\\\\*\\\\W*pgwatch_generated\\\\W*\\\\*/' else true end\n window w as (partition by tag_data->>'queryid' order by time)\n) x\nwhere calls > calls_lag\ngroup by 2, 3\norder by 1 desc nulls last\nlimit $top\n) z\norder by 1 desc nulls last", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top queries by avg. runtime", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "", + "fontSize": "90%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 7, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "link": false, + "linkTargetBlank": true, + "linkTooltip": "", + "linkUrl": "", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Query ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Single query details' dashboard for that queryid", + "linkUrl": "/d/single-query-details?orgId=1&var-dbname=$dbname&var-queryid=$__cell&from=$__from&to=$__to", + "pattern": "queryid", + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Calls ", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Query", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": true, + "linkTooltip": "Use 'edit' mode to be able to copy the query text", + "linkUrl": "", + "pattern": "query", + "preserveFormat": false, + "sanitize": true, + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "alias": "ass", + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n top,\n queryid,\n (select data->>'users' from stat_statements where $__timeFilter(time) and dbname = '$dbname' and tag_data->>'queryid' = queryid order by time desc limit 1) as \"Users\",\n query\nfrom (\nselect\n sum(calls - calls_lag)::int8 as top,\n queryid,\n case when length(query) > 200 then query::varchar(200) || '...' else query end as query\nfrom (\n select\n tag_data->>'queryid' as queryid, tag_data->>'query' as query, data->>'users' as users,\n (data->>'calls')::int8 as calls, lag((data->>'calls')::int8) over w as calls_lag,\n time\n from stat_statements\n where dbname = '$dbname'\n and $__timeFilter(time)\n and case when '$hide_pgwatch_generated' = 'yes' then not tag_data->>'query' ~* E'\\\\(extract\\\\(\\\\$\\\\d+\\\\W*from\\\\W*now\\\\(\\\\)\\\\)\\\\W?\\\\*\\\\W*\\\\$\\\\d+\\\\).*::\\\\w+\\\\W+as\\\\W+epoch_ns\\\\W*,'\n and not tag_data->>'query' ~* E'/\\\\*\\\\W*pgwatch_generated\\\\W*\\\\*/' else true end \n window w as (partition by tag_data->>'queryid' order by time)\n) x\nwhere calls > calls_lag\ngroup by 2, 3\norder by 1 desc nulls last\nlimit $top\n) z\norder by 1 desc nulls last", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top queries by calls", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Does not include Shared Buffer activities. Same as 'By total runtime' if no direct IO was performed. Requires track_io_timing=on", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 8, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "IO time", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 1, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "ms" + }, + { + "alias": "Query ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Opens 'Single query details' dashboard for that queryid", + "linkUrl": "/d/single-query-details?orgId=1&var-dbname=$dbname&var-queryid=$__cell&from=$__from&to=$__to", + "pattern": "queryid", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Query", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": true, + "linkTooltip": "Use 'edit' mode to be able to copy the query text", + "linkUrl": "", + "pattern": "query", + "sanitize": true, + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT top(\"spread\", \"queryid\", \"query\", $top) FROM (SELECT spread(\"blk_read_time\") + spread(\"blk_write_time\") AS spread FROM \"stat_statements\" WHERE $timeFilter AND \"dbname\" = '$dbname' GROUP BY time(3650d), \"queryid\", \"query\" fill(none) ) WHERE spread > 0", + "rawQuery": true, + "rawSql": "select\n top,\n queryid,\n (select data->>'users' from stat_statements where $__timeFilter(time) and dbname = '$dbname' and tag_data->>'queryid' = queryid order by time desc limit 1) as \"Users\",\n query\nfrom (\nselect\n sum((wt - wt_lag) + (rt - rt_lag))::int8 as top,\n queryid,\n case when length(query) > 200 then query::varchar(200) || '...' else query end as query\nfrom (\n select\n tag_data->>'queryid' as queryid, tag_data->>'query' as query, data->>'users' as users,\n (data->>'blk_read_time')::float8 as rt, lag((data->>'blk_read_time')::float8) over w as rt_lag,\n (data->>'blk_write_time')::float8 as wt, lag((data->>'blk_write_time')::float8) over w as wt_lag, \n time\n from stat_statements\n where dbname = '$dbname'\n and $__timeFilter(time)\n and case when '$hide_pgwatch_generated' = 'yes' then not tag_data->>'query' ~* E'\\\\(extract\\\\(\\\\$\\\\d+\\\\W*from\\\\W*now\\\\(\\\\)\\\\)\\\\W?\\\\*\\\\W*\\\\$\\\\d+\\\\).*::\\\\w+\\\\W+as\\\\W+epoch_ns\\\\W*,'\n and not tag_data->>'query' ~* E'/\\\\*\\\\W*pgwatch_generated\\\\W*\\\\*/' else true end \n window w as (partition by tag_data->>'queryid' order by time)\n) x\nwhere rt > rt_lag or wt > wt_lag\ngroup by 2, 3\norder by 1 desc nulls last\nlimit $top\n) z\norder by 1 desc nulls last", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top queries by direct (backend) IO", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Shared buffer + temp buffer reading / writing", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 10, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Block bandwith", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 1, + "pattern": "top", + "thresholds": [], + "type": "number", + "unit": "decbytes" + }, + { + "alias": "Query ID", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Single query details' dashboard for that queryid", + "linkUrl": "/d/single-query-details?orgId=1&var-dbname=$dbname&var-queryid=$__cell&from=$__from&to=$__to", + "pattern": "queryid", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Query", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": true, + "linkTooltip": "Use 'edit' mode to be able to copy the query text", + "linkUrl": "", + "pattern": "query", + "sanitize": true, + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT top(\"spread\", \"queryid\", \"query\", $top) FROM (SELECT spread(\"blk_read_time\") + spread(\"blk_write_time\") AS spread FROM \"stat_statements\" WHERE $timeFilter AND \"dbname\" = '$dbname' GROUP BY time(3650d), \"queryid\", \"query\" fill(none) ) WHERE spread > 0", + "rawQuery": true, + "rawSql": "select\n top,\n queryid,\n (select data->>'users' from stat_statements where $__timeFilter(time) and dbname = '$dbname' and tag_data->>'queryid' = queryid order by time desc limit 1) as \"Users\",\n query\nfrom (\nselect\n sum(blocks_touched - blocks_touched_lag)*8192 /*assuming default 8k block size*/ as top,\n queryid,\n case when length(query) > 200 then query::varchar(200) || '...' else query end as query\nfrom (\n select\n tag_data->>'queryid' as queryid, tag_data->>'query' as query, data->>'users' as users,\n (data->>'shared_blks_hit')::int8 + (data->>'shared_blks_read')::int8 + (data->>'shared_blks_dirtied')::int8 + (data->>'shared_blks_written')::int8\n + (data->>'temp_blks_read')::int8 + (data->>'temp_blks_written')::int8 as blocks_touched,\n lag((data->>'shared_blks_hit')::int8 + (data->>'shared_blks_read')::int8 + (data->>'shared_blks_dirtied')::int8 + (data->>'shared_blks_written')::int8\n + (data->>'temp_blks_read')::int8 + (data->>'temp_blks_written')::int8) over w as blocks_touched_lag,\n time\n from stat_statements\n where dbname = '$dbname'\n and $__timeFilter(time)\n and case when '$hide_pgwatch_generated' = 'yes' then not tag_data->>'query' ~* E'\\\\(extract\\\\(\\\\$\\\\d+\\\\W*from\\\\W*now\\\\(\\\\)\\\\)\\\\W?\\\\*\\\\W*\\\\$\\\\d+\\\\).*::\\\\w+\\\\W+as\\\\W+epoch_ns\\\\W*,'\n and not tag_data->>'query' ~* E'/\\\\*\\\\W*pgwatch_generated\\\\W*\\\\*/' else true end \n window w as (partition by tag_data->>'queryid' order by time)\n) x\nwhere blocks_touched >= blocks_touched_lag\ngroup by 2, 3\nhaving sum(blocks_touched - blocks_touched_lag) > 0\norder by 1 desc nulls last\nlimit $top\n) z\norder by 1 desc nulls last", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top by block bandwith (assuming 8K blocks)", + "transform": "table", + "type": "table" + }, + { + "content": "Brought to you by: \"Cybertec", + "datasource": null, + "editable": true, + "error": false, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 4, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 22, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "test", + "value": "test" + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'stat_statements' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "tags": [], + "text": "5", + "value": "5" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "top", + "options": [ + { + "selected": false, + "text": "1", + "value": "1" + }, + { + "selected": false, + "text": "3", + "value": "3" + }, + { + "selected": true, + "text": "5", + "value": "5" + }, + { + "selected": false, + "text": "10", + "value": "10" + }, + { + "selected": false, + "text": "15", + "value": "15" + }, + { + "selected": false, + "text": "20", + "value": "20" + }, + { + "selected": false, + "text": "30", + "value": "30" + }, + { + "selected": false, + "text": "40", + "value": "40" + }, + { + "selected": false, + "text": "50", + "value": "50" + } + ], + "query": "1,3,5,10,15,20,30,40,50", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "current": { + "tags": [], + "text": "yes", + "value": "yes" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "hide_pgwatch_generated", + "options": [ + { + "selected": true, + "text": "yes", + "value": "yes" + }, + { + "selected": false, + "text": "no", + "value": "no" + } + ], + "query": "yes,no", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Stat statements Top", + "uid": "stat-statements-top", + "version": 1 + } + system-stats-time-lag.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((data->'cpu_utilization')::float8) \"CPU utilization\"\nFROM\n psutil_cpu\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1\nORDER BY 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg(cpu_utilization) \"CPU utilization-$lag_interval\"\nFROM (\nSELECT\n time + '$lag_interval'::interval as time,\n (data->'cpu_utilization')::float8 as cpu_utilization\nFROM\n psutil_cpu\nWHERE\n dbname = '$dbname'\n and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n) x\nGROUP BY 1\nORDER BY 1\n", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU utilization", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((data->'iowait')::float8) as \"IOWait\"\nFROM\n psutil_cpu\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1\nORDER BY 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg(iowait) \"IOWait-$lag_interval\"\nFROM (\nSELECT\n time + '$lag_interval'::interval as time,\n (data->'iowait')::float8 as iowait\nFROM\n psutil_cpu\nWHERE\n dbname = '$dbname'\n and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n) x\nGROUP BY 1\nORDER BY 1\n", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IO Wait", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "(total - available) / total * 100", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((data->'percent')::float8) as \"Mem. used\"\nFROM\n psutil_mem\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1\nORDER BY 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg(used) \"Mem. used-$lag_interval\"\nFROM (\nSELECT\n time + '$lag_interval'::interval as time,\n (data->'percent')::float8 as used\nFROM\n psutil_mem\nWHERE\n dbname = '$dbname'\n and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n) x\nGROUP BY 1\nORDER BY 1\n", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory used (%)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((data->'available')::float8) as \"Mem. available\"\nFROM\n psutil_mem\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1\nORDER BY 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg(available) \"Mem. available-$lag_interval\"\nFROM (\nSELECT\n time + '$lag_interval'::interval as time,\n (data->'available')::float8 as available\nFROM\n psutil_mem\nWHERE\n dbname = '$dbname'\n and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n) x\nGROUP BY 1\nORDER BY 1\n", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory available", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((data->'swap_percent')::float8) as \"Swap used\"\nFROM\n psutil_mem\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1\nORDER BY 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg(swap_percent) \"Swap used-$lag_interval\"\nFROM (\nSELECT\n time + '$lag_interval'::interval as time,\n (data->'swap_percent')::float8 as swap_percent\nFROM\n psutil_mem\nWHERE\n dbname = '$dbname'\n and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n) x\nGROUP BY 1\nORDER BY 1\n", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Swap used (%)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((read_bytes_lag - read_bytes) / extract(epoch from time_lag - time))::int8 as \"read_bytes_s\"\nFROM (\n SELECT\n time,\n lag(time) over w as time_lag,\n (data->'read_bytes')::int8 as read_bytes,\n lag((data->'read_bytes')::int8) over w as read_bytes_lag\n FROM\n psutil_disk_io_total\n WHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n WINDOW w AS (ORDER BY time)\n) x\nWHERE read_bytes >= read_bytes_lag and time > time_lag\nGROUP BY 1\nORDER BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((read_bytes_lag - read_bytes) / extract(epoch from time_lag - time))::int8 as \"read_bytes_s-$lag_interval\"\nFROM (\n SELECT\n time + '$lag_interval'::interval as time,\n lag(time) over w + '$lag_interval'::interval as time_lag,\n (data->'read_bytes')::int8 as read_bytes,\n lag((data->'read_bytes')::int8) over w as read_bytes_lag\n FROM\n psutil_disk_io_total\n WHERE\n time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n AND dbname = '$dbname'\n WINDOW w AS (ORDER BY time)\n) x\nWHERE read_bytes >= read_bytes_lag and time > time_lag\nGROUP BY 1\nORDER BY 1\n", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total bytes read per second", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 42 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((write_bytes_lag - write_bytes) / extract(epoch from time_lag - time))::int8 as \"write_bytes_s\"\nFROM (\n SELECT\n time,\n lag(time) over w as time_lag,\n (data->'write_bytes')::int8 as write_bytes,\n lag((data->'write_bytes')::int8) over w as write_bytes_lag\n FROM\n psutil_disk_io_total\n WHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n WINDOW w AS (ORDER BY time)\n) x\nWHERE write_bytes >= write_bytes_lag and time > time_lag\nGROUP BY 1\nORDER BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((write_bytes_lag - write_bytes) / extract(epoch from time_lag - time))::int8 as \"write_bytes_s-$lag_interval\"\nFROM (\n SELECT\n time + '$lag_interval'::interval as time,\n lag(time) over w + '$lag_interval'::interval as time_lag,\n (data->'write_bytes')::int8 as write_bytes,\n lag((data->'write_bytes')::int8) over w as write_bytes_lag\n FROM\n psutil_disk_io_total\n WHERE\n time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n AND dbname = '$dbname'\n WINDOW w AS (ORDER BY time)\n) x\nWHERE write_bytes >= write_bytes_lag and time > time_lag\nGROUP BY 1\nORDER BY 1\n", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total bytes written per second", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 22, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "datasource": null, + "definition": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric LIKE 'psutil%' ORDER BY 1;", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric LIKE 'psutil%' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "30m", + "value": "30m" + }, + "hide": 0, + "label": null, + "name": "agg_interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": true, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1m,5m,10m,15m,30m,1h,6h,12h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "1d", + "value": "1d" + }, + "hide": 0, + "label": null, + "name": "lag_interval", + "options": [ + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": true, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Systems stats time lag comparison", + "uid": "systems-stats-time-lag-comparison", + "variables": { + "list": [] + }, + "version": 1 + } + system-stats.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Needs according psutil_* helper stored procedures installed on monitored Postgres DBs", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": { + "load (1min) normalized to #CPU": "#890f02", + "load 1min norm.": "#bf1b00", + "load_1min_norm": "#bf1b00", + "normalized to #CPU": "#890f02", + "utilization": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "a.k.a. CPU run queue length or Load Average, reported e.g. by the \"uptime\" command", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "interval": "2m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/normalized/", + "linewidth": 4 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "normalized to #CPU", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "psutil_cpu", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((data->'load_1m_norm')::float8) \"normalized to #CPU\",\n avg((data->'load_1m')::float8) \"as reported by /proc/loadavg\"\nFROM\n psutil_cpu\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1\nORDER BY 1\n", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "load_1m_norm" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "System load ($agg_interval avg.)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "none", + "label": "As reported by /proc/loadavg ", + "logBase": 10, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "load_1min_norm": "#bf1b00", + "utilization": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "System-wide CPU utilization as reported by psutil.cpu_percent()", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 3, + "interval": "2m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "utilization", + "yaxis": 1 + }, + { + "alias": "load_1min_norm", + "yaxis": 2 + }, + { + "alias": "load 1min norm.", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "utilization", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "psutil_cpu", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((data->'cpu_utilization')::float8) \"utilization\"\nFROM\n psutil_cpu\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1\nORDER BY 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "cpu_utilization" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU utilization % ($agg_interval avg.)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 2, + "interval": "2m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "idle", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "hide": false, + "measurement": "psutil_cpu", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((data->'idle')::float8) as \"idle\",\n avg((data->'user')::float8) as \"user\",\n avg((data->'system')::float8) as \"system\",\n avg((data->'iowait')::float8) as \"iowait\",\n avg((data->'other')::float8) as \"other\",\n avg((data->'irqs')::float8) as \"irqs\"\nFROM\n psutil_cpu\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1\nORDER BY 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "idle" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU usage distribution ($agg_interval avg.)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "Doesn't have to necessarily equal 100% to values reported by the \"free\" command - see [here](https://psutil.readthedocs.io/en/latest/#psutil.virtual_memory) for more", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 5, + "interval": "2m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "used", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "psutil_mem", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((data->'used')::float8) as \"used\",\n avg((data->'total')::float8) as \"total\",\n avg((data->'available')::float8) as \"available\",\n avg((data->'free')::float8) as \"free\",\n avg((data->'buff_cache')::float8) as \"buff_cache\"\nFROM\n psutil_mem\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1\nORDER BY 1\n", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "used" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory usage ($agg_interval avg.)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "Doesn't have to necessarily equal 100% to values reported by the \"free\" command - see [here](https://psutil.readthedocs.io/en/latest/#psutil.virtual_memory) for more", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 6, + "interval": "2m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "swap_used", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "psutil_mem", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((data->'swap_used')::float8) as \"swap_used\",\n avg((data->'swap_total')::float8) as \"swap_total\"\nFROM\n psutil_mem\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1\nORDER BY 1\n", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "swap_used" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Swap usage ($agg_interval avg.)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "For all disks on the host, not only Postgres related", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 11, + "interval": "2m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "read_bytes", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "psutil_disk_io_total", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT * FROM (\nSELECT\n $__timeGroup(time, $agg_interval),\n max((data->'read_bytes')::int8) - min((data->'read_bytes')::int8) as \"read_bytes\",\n max((data->'write_bytes')::int8) - min((data->'write_bytes')::int8) as \"write_bytes\"\nFROM\n psutil_disk_io_total\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1\n) x WHERE read_bytes >= 0 and write_bytes >= 0\nORDER BY 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "read_bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + }, + { + "params": [ + "1s" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Host disk IO Totals ($agg_interval aggregate)", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "For Postgres data / WAL / log folder partitions and user defined tablespace partitions", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 7, + "interval": "2m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_dir_or_tablespace]] ([[tag_path]])", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "dir_or_tablespace" + ], + "type": "tag" + }, + { + "params": [ + "path" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "psutil_disk", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n (tag_data->>'dir_or_tablespace') || ' (' || (tag_data->>'path') || ')',\n avg((data->'percent')::float8) as \"percent\"\nFROM\n psutil_disk\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1, 2\nORDER BY 1, 2", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "percent" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk usage %", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 1, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "For Postgres data / WAL / log folder partitions and user defined tablespace partitions", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 8, + "interval": "2m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "[[tag_dir_or_tablespace]] ([[tag_path]])", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "dir_or_tablespace" + ], + "type": "tag" + }, + { + "params": [ + "path" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "psutil_disk", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n (tag_data->>'dir_or_tablespace') || ' (' || (tag_data->>'path') || ')',\n avg((data->'free')::float8) as \"free\"\nFROM\n psutil_disk\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1, 2\nORDER BY 1, 2", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "free" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk space available", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 1, + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "Brought to you by: \"Cybertec", + "editable": true, + "error": false, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 10, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": null, + "text": null + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric LIKE 'psutil%' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "10m", + "value": "10m" + }, + "hide": 0, + "label": null, + "name": "agg_interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": true, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1m,5m,10m,15m,30m,1h,6h,12h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "System Stats", + "uid": "system-stats", + "version": 1 + } + table-details-time-lag.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n sum(ins-ins_lag) as \"INSERT\"\nfrom (\n select \n (data->'n_tup_ins')::int8 as ins,\n lag((data->'n_tup_ins')::int8) over w as ins_lag,\n time\n from\n table_stats\n where\n dbname = '$dbname'\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n and $__timeFilter(time)\n window w as (order by time)\n\n) x\nwhere ins >= ins_lag\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n sum(ins-ins_lag) as \"INSERT-$lag_interval\"\nfrom (\n select \n (data->'n_tup_ins')::int8 as ins,\n lag((data->'n_tup_ins')::int8) over w as ins_lag,\n time + '$lag_interval'::interval as time\n from\n table_stats\n where\n dbname = '$dbname'\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n window w as (order by time)\n\n) x\nwhere ins >= ins_lag\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "INSERT-s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n sum(upd-upd_lag) as \"UPDATE\"\nfrom (\n select \n (data->'n_tup_upd')::int8 as upd,\n lag((data->'n_tup_upd')::int8) over w as upd_lag,\n time\n from\n table_stats\n where\n dbname = '$dbname'\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n and $__timeFilter(time)\n window w as (order by time)\n\n) x\nwhere upd >= upd_lag\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n sum(upd-upd_lag) as \"UPDATE-$lag_interval\"\nfrom (\n select \n (data->'n_tup_upd')::int8 as upd,\n lag((data->'n_tup_upd')::int8) over w as upd_lag,\n time + '$lag_interval'::interval as time\n from\n table_stats\n where\n dbname = '$dbname'\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n window w as (order by time)\n\n) x\nwhere upd >= upd_lag\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "UPDATE-s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n sum(del-del_lag) as \"DELETE\"\nfrom (\n select \n (data->'n_tup_del')::int8 as del,\n lag((data->'n_tup_del')::int8) over w as del_lag,\n time\n from\n table_stats\n where\n dbname = '$dbname'\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n and $__timeFilter(time)\n window w as (order by time)\n\n) x\nwhere del >= del_lag\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n sum(del-del_lag) as \"DELETE-$lag_interval\"\nfrom (\n select \n (data->'n_tup_del')::int8 as del,\n lag((data->'n_tup_del')::int8) over w as del_lag,\n time + '$lag_interval'::interval as time\n from\n table_stats\n where\n dbname = '$dbname'\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n window w as (order by time)\n\n) x\nwhere del >= del_lag\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DELETE-s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n sum(seq_scan-seq_scan_lag) as seq_scan\nFROM ( \n SELECT\n (data->'seq_scan')::int8 as seq_scan, lag((data->'seq_scan')::int8) over w as seq_scan_lag,\n time\n FROM\n table_stats\n WHERE\n $__timeFilter(time) AND dbname = '$dbname'\n AND tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n WINDOW w as (order by time)\n) x\nWHERE seq_scan >= seq_scan_lag\nGROUP BY 1\nORDER BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n sum(seq_scan-seq_scan_lag) as \"seq_scan-$lag_interval\"\nFROM ( \n SELECT\n (data->'seq_scan')::int8 as seq_scan, lag((data->'seq_scan')::int8) over w as seq_scan_lag,\n time + '$lag_interval'::interval as time\n FROM\n table_stats\n WHERE\n time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n AND dbname = '$dbname'\n AND tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n WINDOW w as (order by time)\n) x\nWHERE seq_scan >= seq_scan_lag\nGROUP BY 1\nORDER BY 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Seq. Scan", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n sum(idx_scan-idx_scan_lag) as idx_scan\nFROM ( \n SELECT\n (data->'idx_scan')::int8 as idx_scan, lag((data->'idx_scan')::int8) over w as idx_scan_lag,\n time\n FROM\n table_stats\n WHERE\n $__timeFilter(time) AND dbname = '$dbname'\n AND tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n WINDOW w as (order by time)\n) x\nWHERE idx_scan >= idx_scan_lag\nGROUP BY 1\nORDER BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n sum(idx_scan-idx_scan_lag) as \"idx_scan-$lag_interval\"\nFROM ( \n SELECT\n (data->'idx_scan')::int8 as idx_scan, lag((data->'idx_scan')::int8) over w as idx_scan_lag,\n time + '$lag_interval'::interval as time\n FROM\n table_stats\n WHERE\n time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n AND dbname = '$dbname'\n AND tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n WINDOW w as (order by time)\n) x\nWHERE idx_scan >= idx_scan_lag\nGROUP BY 1\nORDER BY 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Idx. Scan", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "i.e. the main table excluding indexes and TOAST", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg( case when hh = hh_lag and hr = hr_lag then null else (hh-hh_lag)::numeric * 100 / (hh-hh_lag+hr-hr_lag) end ) as \"Heap hit\"\nfrom (\n select \n (data->'heap_blks_hit')::int8 as hh, lag((data->'heap_blks_hit')::int8) over w as hh_lag,\n (data->'heap_blks_read')::int8 as hr, lag((data->'heap_blks_read')::int8) over w as hr_lag,\n time\n from table_io_stats\n where dbname = '$dbname' and $__timeFilter(time)\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n window w as (order by time)\n) x\nwhere hh >= hh_lag and hr >= hr_lag\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg( case when hh = hh_lag and hr = hr_lag then null else (hh-hh_lag)::numeric * 100 / (hh-hh_lag+hr-hr_lag) end ) as \"Heap hit-$lag_interval\"\nfrom (\n select \n (data->'heap_blks_hit')::int8 as hh, lag((data->'heap_blks_hit')::int8) over w as hh_lag,\n (data->'heap_blks_read')::int8 as hr, lag((data->'heap_blks_read')::int8) over w as hr_lag,\n time +'$lag_interval'::interval as time\n from table_io_stats\n where dbname = '$dbname'\n and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n window w as (order by time)\n) x\nwhere hh >= hh_lag and hr >= hr_lag\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Shared buffers hit rate - Heap", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 54 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n 100 * avg( case when tr = tr_lag and th = th_lag and tir = tir_lag and tih = tir_lag then null else (coalesce(th,0)-coalesce(th_lag,0) + coalesce(tih,0)-coalesce(tih_lag,0))::numeric / (coalesce(th,0)-coalesce(th_lag,0) + coalesce(tr,0)-coalesce(tr_lag,0) + coalesce(tir,0)-coalesce(tir_lag,0) + coalesce(tih,0)-coalesce(tih_lag,0)) end ) as \"Toast hit\"\nfrom (\n select \n (data->'toast_blks_read')::int8 as tr, lag((data->'toast_blks_read')::int8) over w as tr_lag,\n (data->'toast_blks_hit')::int8 as th, lag((data->'toast_blks_hit')::int8) over w as th_lag,\n (data->'tidx_blks_read')::int8 as tir, lag((data->'tidx_blks_read')::int8) over w as tir_lag,\n (data->'tidx_blks_hit')::int8 as tih, lag((data->'tidx_blks_hit')::int8) over w as tih_lag,\n time\n from table_io_stats\n where dbname = '$dbname' and $__timeFilter(time)\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n window w as (order by time)\n) x\nwhere th > th_lag or tih > tih_lag or tr > tr_lag or tir > tir_lag\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n 100 * avg( case when tr = tr_lag and th = th_lag and tir = tir_lag and tih = tir_lag then null else (coalesce(th,0)-coalesce(th_lag,0) + coalesce(tih,0)-coalesce(tih_lag,0))::numeric / (coalesce(th,0)-coalesce(th_lag,0) + coalesce(tr,0)-coalesce(tr_lag,0) + coalesce(tir,0)-coalesce(tir_lag,0) + coalesce(tih,0)-coalesce(tih_lag,0)) end ) as \"Toast hit-$lag_interval\"\nfrom (\n select \n (data->'toast_blks_read')::int8 as tr, lag((data->'toast_blks_read')::int8) over w as tr_lag,\n (data->'toast_blks_hit')::int8 as th, lag((data->'toast_blks_hit')::int8) over w as th_lag,\n (data->'tidx_blks_read')::int8 as tir, lag((data->'tidx_blks_read')::int8) over w as tir_lag,\n (data->'tidx_blks_hit')::int8 as tih, lag((data->'tidx_blks_hit')::int8) over w as tih_lag,\n time + '$lag_interval'::interval as time\n from table_io_stats\n where dbname = '$dbname'\n and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n window w as (order by time)\n) x\nwhere th > th_lag or tih > tih_lag or tr > tr_lag or tir > tir_lag\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Shared buffers hit rate - Toast", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "Assuming 8k blocks. Hit + read", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 63 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n 8192 * sum(hh-hh_lag+hr-hr_lag) as \"Heap read\"\nfrom (\n select \n (data->'heap_blks_hit')::int8 as hh, lag((data->'heap_blks_hit')::int8) over w as hh_lag,\n (data->'heap_blks_read')::int8 as hr, lag((data->'heap_blks_read')::int8) over w as hr_lag,\n time\n from table_io_stats\n where dbname = '$dbname' and $__timeFilter(time)\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n window w as (order by time)\n) x\nwhere hr >= hr_lag and hh >= hh_lag\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n 8192 * sum(hh-hh_lag+hr-hr_lag) as \"Heap read-$lag_interval\"\nfrom (\n select \n (data->'heap_blks_hit')::int8 as hh, lag((data->'heap_blks_hit')::int8) over w as hh_lag,\n (data->'heap_blks_read')::int8 as hr, lag((data->'heap_blks_read')::int8) over w as hr_lag,\n time + '$lag_interval'::interval as time\n from table_io_stats\n where dbname = '$dbname'\n and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n window w as (order by time)\n) x\nwhere hr >= hr_lag and hh >= hh_lag\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Block bandwidth - Heap", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "Assuming 8k blocks. Hit + read", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 72 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n 8192 * sum(ih-ih_lag+ir-ir_lag) as \"Idx read\"\nfrom (\n select \n (data->'idx_blks_hit')::int8 as ih, lag((data->'idx_blks_hit')::int8) over w as ih_lag,\n (data->'idx_blks_read')::int8 as ir, lag((data->'idx_blks_read')::int8) over w as ir_lag,\n time\n from table_io_stats\n where dbname = '$dbname' and $__timeFilter(time)\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n window w as (order by time)\n) x\nwhere ir >= ir_lag and ih >= ih_lag\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n 8192 * sum(ih-ih_lag+ir-ir_lag) as \"Idx read-$lag_interval\"\nfrom (\n select \n (data->'idx_blks_hit')::int8 as ih, lag((data->'idx_blks_hit')::int8) over w as ih_lag,\n (data->'idx_blks_read')::int8 as ir, lag((data->'idx_blks_read')::int8) over w as ir_lag,\n time + '$lag_interval'::interval as time\n from table_io_stats\n where dbname = '$dbname'\n and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n window w as (order by time)\n) x\nwhere ir >= ir_lag and ih >= ih_lag\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Block bandwidth - Indexes", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "description": "Assuming 8k blocks. Hit + read", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 81 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n 8192 * sum(coalesce(th,0)-coalesce(th_lag,0) + coalesce(tr,0)-coalesce(tr_lag,0) + coalesce(tir,0)-coalesce(tir_lag,0) + coalesce(tih,0)-coalesce(tih_lag,0)) as \"Toast read\"\nfrom (\n select \n (data->'toast_blks_read')::int8 as tr, lag((data->'toast_blks_read')::int8) over w as tr_lag,\n (data->'toast_blks_hit')::int8 as th, lag((data->'toast_blks_hit')::int8) over w as th_lag,\n (data->'tidx_blks_read')::int8 as tir, lag((data->'tidx_blks_read')::int8) over w as tir_lag,\n (data->'tidx_blks_hit')::int8 as tih, lag((data->'tidx_blks_hit')::int8) over w as tih_lag,\n time\n from table_io_stats\n where dbname = '$dbname' and $__timeFilter(time)\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n window w as (order by time)\n) x\nwhere th > th_lag or tih > tih_lag or tr > tr_lag or tir > tir_lag\ngroup by 1\norder by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n 8192 * sum(coalesce(th,0)-coalesce(th_lag,0) + coalesce(tr,0)-coalesce(tr_lag,0) + coalesce(tir,0)-coalesce(tir_lag,0) + coalesce(tih,0)-coalesce(tih_lag,0)) as \"Toast read-$lag_interval\"\nfrom (\n select \n (data->'toast_blks_read')::int8 as tr, lag((data->'toast_blks_read')::int8) over w as tr_lag,\n (data->'toast_blks_hit')::int8 as th, lag((data->'toast_blks_hit')::int8) over w as th_lag,\n (data->'tidx_blks_read')::int8 as tir, lag((data->'tidx_blks_read')::int8) over w as tir_lag,\n (data->'tidx_blks_hit')::int8 as tih, lag((data->'tidx_blks_hit')::int8) over w as tih_lag,\n time + '$lag_interval'::interval as time\n from table_io_stats\n where dbname = '$dbname'\n and time between ($__timeFrom()::timestamptz - '$lag_interval'::interval) and ($__timeTo()::timestamptz - '$lag_interval'::interval)\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n window w as (order by time)\n) x\nwhere th > th_lag or tih > tih_lag or tr > tr_lag or tir > tir_lag\ngroup by 1\norder by 1", + "refId": "B", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Block bandwidth - Toast", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 22, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": null, + "value": null + }, + "datasource": null, + "definition": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics ORDER BY 1;", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": null, + "definition": "SELECT DISTINCT tag_data->>'table_full_name' FROM table_stats WHERE time > now() - '1d'::interval AND dbname = '$dbname' ORDER BY 1", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": false, + "name": "table_full_name", + "options": [], + "query": "SELECT DISTINCT tag_data->>'table_full_name' FROM table_stats WHERE time > now() - '1d'::interval AND dbname = '$dbname' ORDER BY 1", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "10m", + "value": "10m" + }, + "hide": 0, + "label": null, + "name": "agg_interval", + "options": [ + { + "selected": false, + "text": "1s", + "value": "1s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "2m", + "value": "2m" + }, + { + "selected": false, + "text": "3m", + "value": "3m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": true, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + } + ], + "query": "1s,1m,2m,3m,5m,10m,15m,30m,1h,6h,12h,1d,7d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "1d", + "value": "1d" + }, + "hide": 0, + "label": null, + "name": "lag_interval", + "options": [ + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": true, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-12h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Table details time lag comparison", + "uid": "table-details-time-lag", + "variables": { + "list": [] + }, + "version": 1 + } + table-details.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "total_relation_size", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"total_relation_size_b\") FROM \"table_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND \"table_full_name\" =~ /^$table_full_name$/ AND $timeFilter GROUP BY time($interval) fill(none)", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg((data->'table_size_b')::int8) as table_size,\n avg((data->'total_relation_size_b')::int8) as total_relation_size,\n avg((data->'toast_size_b')::int8) as toast_size\nFROM\n table_stats\nWHERE\n $__timeFilter(time) AND dbname = '$dbname'\n AND tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\nGROUP BY 1\nORDER BY 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "total_relation_size_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + }, + { + "condition": "AND", + "key": "table_full_name", + "operator": "=~", + "value": "/^$table_full_name$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "idx_scans": "#629E51", + "seq_scans": "#BF1B00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 2, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "seq_scans", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "SELECT\n $__timeGroup(time, $agg_interval),\n avg( ((seq_scan-seq_scan_lag)::numeric*3600) / extract(epoch from time - time_lag) ) as seq_scan,\n avg( ((idx_scan-idx_scan_lag)::numeric*3600) / extract(epoch from time - time_lag) ) as idx_scan\nFROM ( \n SELECT\n (data->'seq_scan')::int8 as seq_scan, lag((data->'seq_scan')::int8) over w as seq_scan_lag,\n (data->'idx_scan')::int8 as idx_scan, lag((data->'idx_scan')::int8) over w as idx_scan_lag,\n time, lag(time) over w as time_lag \n FROM\n table_stats\n WHERE\n $__timeFilter(time) AND dbname = '$dbname'\n AND tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n WINDOW w as (order by time)\n) x\nWHERE seq_scan >= seq_scan_lag and time > time_lag\nGROUP BY 1\nORDER BY 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "seq_scan" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1m" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + }, + { + "condition": "AND", + "key": "table_full_name", + "operator": "=~", + "value": "/^$table_full_name$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scans (1h rate, $agg_interval avg.)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 3, + "interval": "5m", + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "INS", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg((ins-ins_lag) * 3600 / extract(epoch from time - time_lag)) as \"INSERT\",\n avg((upd-upd_lag) * 3600 / extract(epoch from time - time_lag)) as \"UPDATE\",\n avg((del-del_lag) * 3600 / extract(epoch from time - time_lag)) as \"DELETE\"\nfrom (\n select \n (data->'n_tup_ins')::int8 as ins, lag((data->'n_tup_ins')::int8) over w as ins_lag,\n (data->'n_tup_upd')::int8 as upd, lag((data->'n_tup_upd')::int8) over w as upd_lag,\n (data->'n_tup_del')::int8 as del, lag((data->'n_tup_del')::int8) over w as del_lag,\n time, lag(time) over w as time_lag\n from table_stats\n where dbname = '$dbname' and $__timeFilter(time)\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n window w as (order by time)\n\n) x\nwhere ins >= ins_lag and time > time_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "n_tup_ins" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1m" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + }, + { + "condition": "AND", + "key": "table_full_name", + "operator": "=~", + "value": "/^$table_full_name$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IUD (1h rate, $agg_interval avg.)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 4, + "interval": "5m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Heap", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "table_io_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT non_negative_derivative(mean(\"heap_blks_hit\"), 10s) / (non_negative_derivative(mean(\"heap_blks_hit\"), 10s) + non_negative_derivative(mean(\"heap_blks_read\"), 10s)) * 100 FROM \"table_io_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND \"table_full_name\" =~ /^$table_full_name$/ AND $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n avg( case when hh = hh_lag and hr = hr_lag then null else (hh-hh_lag)::numeric * 100 / (hh-hh_lag+hr-hr_lag) end ) as \"Heap\",\n avg( case when ih = ih_lag and ir = ir_lag then null else (ih-ih_lag)::numeric * 100 / (ih-ih_lag+ir-ir_lag) end ) as \"Indexes\" \nfrom (\n select \n (data->'heap_blks_hit')::int8 as hh, lag((data->'heap_blks_hit')::int8) over w as hh_lag,\n (data->'heap_blks_read')::int8 as hr, lag((data->'heap_blks_read')::int8) over w as hr_lag,\n (data->'idx_blks_hit')::int8 as ih, lag((data->'idx_blks_hit')::int8) over w as ih_lag,\n (data->'idx_blks_read')::int8 as ir, lag((data->'idx_blks_read')::int8) over w as ir_lag,\n time\n from table_io_stats\n where dbname = '$dbname' and $__timeFilter(time)\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n window w as (order by time)\n) x\nwhere hh >= hh_lag\ngroup by 1\norder by 1", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "heap_blks_hit" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "dbname", + "operator": "=~", + "value": "/^$dbname$/" + }, + { + "condition": "AND", + "key": "table_full_name", + "operator": "=~", + "value": "/^$table_full_name$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Shared Buffers hit rates ($agg_interval avg.)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percent", + "label": null, + "logBase": 1, + "max": "100", + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "description": "", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 20 + }, + "height": "175", + "id": 5, + "interval": "5m", + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_index_name", + "format": "time_series", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "index_name" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "index_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "rawQuery": true, + "rawSql": "select\n $__timeGroup(time, $agg_interval),\n index_name,\n avg( ((idx_scan - idx_scan_lag)::numeric * 3600) / extract(epoch from time - time_lag) )\nfrom (\n select \n (data->'idx_scan')::int8 as idx_scan, lag((data->'idx_scan')::int8) over w as idx_scan_lag,\n tag_data->>'index_name' as index_name,\n time, lag(time) over w as time_lag\n from index_stats\n where dbname = '$dbname' and $__timeFilter(time)\n and tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\n window w as (order by time)\n) x\nwhere idx_scan >= idx_scan_lag and time > time_lag\ngroup by 1, 2\norder by 1, 2", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "idx_scan" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "1m" + ], + "type": "non_negative_derivative" + } + ] + ], + "tags": [ + { + "key": "table_full_name", + "operator": "=~", + "value": "/^$table_full_name$/" + } + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Index Scans per index (1h rate, $agg_interval avg.)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": null, + "description": "Including manual VACUUM and AUTOVACUUM. 1 month WARNING threshold / 2 month ERROR threshold by default", + "format": "dtdurations", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 8, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n (data->'seconds_since_last_vacuum')::int8\nFROM\n table_stats\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n AND tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\nORDER BY time DESC\nLIMIT 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "2592000,5184000", + "title": "Time since last VACUUM", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "decimals": null, + "description": "Including manual ANALYZE and AUTOVACUUM induced. 1 month WARNING threshold / 2 month ERROR threshold by default", + "format": "dtdurations", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 9, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n 0 as time,\n (data->'seconds_since_last_analyze')::int8\nFROM\n table_stats\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n AND tag_data @> jsonb_build_object('table_full_name', '$table_full_name')\nORDER BY time DESC\nLIMIT 1\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "2592000,5184000", + "title": "Time since last ANALYZE", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "content": "Brought to you by: \"Cybertec", + "editable": true, + "error": false, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 6, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": null, + "value": null + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'table_stats' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "tags": [], + "text": null, + "value": null + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "table_full_name", + "options": [], + "query": "SELECT DISTINCT tag_data->>'table_full_name' FROM table_stats WHERE time > current_date-3 AND dbname = '$dbname' ORDER BY 1", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "tags": [], + "text": "10m", + "value": "10m" + }, + "hide": 0, + "label": null, + "name": "agg_interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": true, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "15m", + "value": "15m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + } + ], + "query": "1m,5m,10m,15m,30m,1h,6h,12h,1d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Table details", + "uid": "table-details", + "version": 1 + } + tables-top.json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "columns": [], + "datasource": null, + "description": "Most views here assume that there was no stats reset in the selected time range!", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 12, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "total_relation_size", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "data_size", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Table details' dashboard for that table", + "linkUrl": "/d/table-details?var-dbname=${__cell_0}&var-table_full_name=${__cell}&from=$__from&to=$__to", + "mappingType": 1, + "pattern": "table_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "/^last_/", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": null, + "rawQuery": true, + "rawSql": "select\n x.*,\n (data->'seconds_since_last_vacuum')::int8 as last_vacuum,\n (data->'seconds_since_last_analyze')::int8 as last_analyze\nfrom (\nSELECT\n dbname,\n tag_data->>'table_full_name' as table_full_name,\n max((data->'total_relation_size_b')::int8) as total_relation_size,\n max((data->'table_size_b')::int8) as data_size\nFROM\n table_stats\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1, 2\nORDER BY 3 DESC NULLS LAST\nLIMIT $top\n) x\njoin lateral (select data from table_stats where dbname = x.dbname and tag_data->>'table_full_name' = x.table_full_name and $__timeFilter(time) order by time desc limit 1) y on true\n/*idea: show only primaries? */", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "total_relation_size_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Biggest (total)", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 13, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 3, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "total_relation_size", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "data_size", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Table details' dashboard for that table", + "linkUrl": "/d/table-details?var-dbname=${__cell_0}&var-table_full_name=${__cell}&from=$__from&to=$__to", + "mappingType": 1, + "pattern": "table_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "/^last_/", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": null, + "rawQuery": true, + "rawSql": "SELECT\n x.*,\n (data->'seconds_since_last_vacuum')::int8 as last_vacuum,\n (data->'seconds_since_last_analyze')::int8 as last_analyze\nFROM (\nSELECT\n dbname,\n tag_data->>'table_full_name' as table_full_name,\n max((data->'total_relation_size_b')::int8) as total_relation_size,\n max((data->'table_size_b')::int8) as data_size\nFROM\n table_stats\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY 1, 2\nORDER BY 4 DESC NULLS LAST\nLIMIT $top\n) x\njoin lateral (select data from table_stats where dbname = x.dbname and tag_data->>'table_full_name' = x.table_full_name and $__timeFilter(time) order by time desc limit 1) y on true\n", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "total_relation_size_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Biggest (data)", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 5, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Table details' dashboard for that table", + "linkUrl": "/d/table-details?var-dbname=${__cell_0}&var-table_full_name=${__cell}&from=$__from&to=$__to", + "mappingType": 1, + "pattern": "table_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "growth", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "total_relation_size", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "/^last_/", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select dbname, table_full_name, top(growth, $top) as growth from (SELECT spread(\"total_relation_size_b\") as growth FROM \"table_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY dbname, table_full_name fill(none))", + "rawQuery": true, + "rawSql": "SELECT\n x.*,\n (data->'seconds_since_last_vacuum')::int8 as last_vacuum,\n (data->'seconds_since_last_analyze')::int8 as last_analyze\nFROM (\n SELECT\n dbname,\n tag_data->>'table_full_name' as table_full_name,\n max((data->'total_relation_size_b')::int8) - min((data->'total_relation_size_b')::int8) as growth,\n max((data->'total_relation_size_b')::int8) as total_relation_size\n FROM\n table_stats\n WHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n GROUP BY 1, 2\n ORDER BY growth DESC NULLS LAST\n LIMIT $top\n) x\njoin lateral (select data from table_stats where dbname = x.dbname and tag_data->>'table_full_name' = x.table_full_name and $__timeFilter(time) order by time desc limit 1) y on true\n\n/*\n-- for fully correct but slow version:\nSELECT\n dbname,\n table_full_name,\n max(last_value) - max(first_value) as growth,\n max(last_value) as total_relation_size,\n max(last_vacuum) as last_vacuum,\n max(last_analyze) as last_analyze\nFROM (\n SELECT\n dbname,\n tag_data->>'table_full_name' as table_full_name,\n first_value((data->'total_relation_size_b')::int8) over w,\n last_value((data->'total_relation_size_b')::int8) over w,\n last_value((data->'seconds_since_last_vacuum')::int8) over w as last_vacuum,\n last_value((data->'seconds_since_last_analyze')::int8) over w as last_analyze\n FROM\n table_stats\n WHERE\n $__timeFilter(time)\n AND dbname IN ($dbname)\n WINDOW w AS (partition by dbname, tag_data->>'table_full_name' order by time rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)\n) x\nGROUP BY 1, 2\nORDER BY 3 DESC NULLS LAST\nLIMIT $top\n*/", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "total_relation_size_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Biggest growth (total relation size)", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 4, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "seq_scan", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "table_size", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Table details dashboard for that table", + "linkUrl": "/d/table-details?var-dbname=${__cell_0}&var-table_full_name=${__cell}&from=$__from&to=$__to", + "mappingType": 1, + "pattern": "table_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "/^last_/", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": null, + "rawQuery": true, + "rawSql": "SELECT\n x.*,\n (data->'table_size_b')::int8 as table_size,\n (data->'seconds_since_last_vacuum')::int8 as last_vacuum,\n (data->'seconds_since_last_analyze')::int8 as last_analyze\nFROM (\nSELECT\n dbname,\n tag_data->>'table_full_name' as table_full_name,\n max((data->'seq_scan')::int8) - min((data->'seq_scan')::int8) as seq_scan\nFROM\n table_stats\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\n AND (data->'table_size_b')::int8 > 100000000\nGROUP BY\n 1, 2\nHAVING\n max((data->'seq_scan')::int8) > min((data->'seq_scan')::int8)\nORDER BY\n 3 DESC\nLIMIT\n $top\n) x\njoin lateral (select data from table_stats where dbname = x.dbname and tag_data->>'table_full_name' = x.table_full_name and $__timeFilter(time) order by time desc limit 1) y on true", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "total_relation_size_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Most \"seq. scans\" (tables > 100MB )", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 6, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Table details' dashboard for that table", + "linkUrl": "/d/table-details?var-dbname=${__cell_0}&var-table_full_name=${__cell}&from=$__from&to=$__to", + "mappingType": 1, + "pattern": "table_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "total_relation_size", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "ins", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "/^last_/", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select dbname, table_full_name, top(ins, $top) as \"ins\" from (select mean(ins) as \"ins\" from (SELECT non_negative_derivative(last(\"n_tup_ins\"), 1h) as \"ins\" FROM \"table_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY time(1h), dbname, table_full_name fill(0)) GROUP BY dbname, table_full_name)", + "rawQuery": true, + "rawSql": "SELECT\n x.*,\n (data->'total_relation_size_b')::int8 as total_relation_size,\n (data->'seconds_since_last_vacuum')::int8 as last_vacuum,\n (data->'seconds_since_last_analyze')::int8 as last_analyze\nFROM (\nSELECT\n dbname,\n tag_data->>'table_full_name' as table_full_name,\n max((data->'n_tup_ins')::int8) - min((data->'n_tup_ins')::int8) as ins\nFROM\n table_stats\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY\n 1, 2\nHAVING\n max((data->'n_tup_ins')::int8) > min((data->'n_tup_ins')::int8)\nORDER BY\n 3 DESC NULLS LAST\nLIMIT\n $top\n) x\njoin lateral (select data from table_stats where dbname = x.dbname and tag_data->>'table_full_name' = x.table_full_name and $__timeFilter(time) order by time desc limit 1) y on true", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "total_relation_size_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Most INSERT-s (aggregate over selected time period)", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 7, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Table details' dashboard for that table", + "linkUrl": "/d/table-details?var-dbname=${__cell_0}&var-table_full_name=${__cell}&from=$__from&to=$__to", + "mappingType": 1, + "pattern": "table_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "total_relation_size", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "upd", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "/^last_/", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select dbname, table_full_name, top(ins, $top) as \"upd\" from (select mean(ins) as \"ins\" from (SELECT non_negative_derivative(last(\"n_tup_upd\"), 1h) as \"ins\" FROM \"table_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY time(1h), dbname, table_full_name fill(0)) GROUP BY dbname, table_full_name)", + "rawQuery": true, + "rawSql": "SELECT\n x.*,\n (data->'total_relation_size_b')::int8 as total_relation_size, \n (data->'seconds_since_last_vacuum')::int8 as last_vacuum,\n (data->'seconds_since_last_analyze')::int8 as last_analyze\nFROM (\nSELECT\n dbname,\n tag_data->>'table_full_name' as table_full_name,\n max((data->'n_tup_upd')::int8) - min((data->'n_tup_upd')::int8) as upd -- assumes no reset\nFROM\n table_stats\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY\n 1, 2\nHAVING\n max((data->'n_tup_upd')::int8) > min((data->'n_tup_upd')::int8)\nORDER BY\n 3 DESC NULLS LAST\nLIMIT\n $top\n) x\njoin lateral (select data from table_stats where dbname = x.dbname and tag_data->>'table_full_name' = x.table_full_name and $__timeFilter(time) order by time desc limit 1) y on true", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "total_relation_size_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Most UPDATE-s (aggregate over selected time period)", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 8, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Table details' dashboard for that table", + "linkUrl": "/d/table-details?var-dbname=${__cell_0}&var-table_full_name=${__cell}&from=$__from&to=$__to", + "mappingType": 1, + "pattern": "table_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "total_relation_size", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "del", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "/^last_/", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select dbname, table_full_name, top(ins, $top) as \"del\" from (select mean(ins) as \"ins\" from (SELECT non_negative_derivative(last(\"n_tup_del\"), 1h) as \"ins\" FROM \"table_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY time(1h), dbname, table_full_name fill(0)) GROUP BY dbname, table_full_name)", + "rawQuery": true, + "rawSql": "SELECT\n x.*,\n (data->'total_relation_size_b')::int8 as total_relation_size, \n (data->'seconds_since_last_vacuum')::int8 as last_vacuum,\n (data->'seconds_since_last_analyze')::int8 as last_analyze\nFROM (\nSELECT\n dbname,\n tag_data->>'table_full_name' as table_full_name,\n max((data->'n_tup_del')::int8) - min((data->'n_tup_del')::int8) as del\nFROM\n table_stats\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY\n 1, 2\nHAVING\n max((data->'n_tup_del')::int8) > min((data->'n_tup_del')::int8)\nORDER BY\n 3 DESC NULLS LAST\nLIMIT\n $top\n) x\njoin lateral (select data from table_stats where dbname = x.dbname and tag_data->>'table_full_name' = x.table_full_name and $__timeFilter(time) order by time desc limit 1) y on true", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "total_relation_size_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Most DELETE-s (aggregate over selected time period)", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": null, + "description": "Based on pg_statio_user_tables. Assumes 8KB page size (change the query if not so). Also note that due to OS file system cache only some reads will hit the disk", + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 11, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "align": "auto", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTargetBlank": true, + "linkTooltip": "Opens 'Table details' dashboard for that table", + "linkUrl": "/d/table-details?var-dbname=${__cell_0}&var-table_full_name=${__cell}&from=$__from&to=$__to", + "mappingType": 1, + "pattern": "table_full_name", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "total_relation_size", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "/^last_/", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 1, + "pattern": "blks_read", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 1, + "mappingType": 1, + "pattern": "data_size", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": null, + "mappingType": 1, + "pattern": "seq_scan", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "format": "table", + "group": [], + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "table_stats", + "metricColumn": "none", + "orderByTime": "ASC", + "policy": "default", + "query": "select dbname, table_full_name, top(ins, $top) as \"del\" from (select mean(ins) as \"ins\" from (SELECT non_negative_derivative(last(\"n_tup_del\"), 1h) as \"ins\" FROM \"table_stats\" WHERE \"dbname\" =~ /^$dbname$/ AND $timeFilter GROUP BY time(1h), dbname, table_full_name fill(0)) GROUP BY dbname, table_full_name)", + "rawQuery": true, + "rawSql": "SELECT\n x.*,\n (data->'table_size_b')::int8 as data_size,\n (data->'seq_scan')::int8 - (select (data->'seq_scan')::int8 from table_stats where dbname = x.dbname and tag_data->>'table_full_name' = x.table_full_name and $__timeFilter(time) order by time limit 1) as seq_scan,\n (data->'seconds_since_last_vacuum')::int8 as last_vacuum,\n (data->'seconds_since_last_analyze')::int8 as last_analyze\nFROM (\nSELECT\n dbname,\n tag_data->>'table_full_name' as table_full_name,\n 8192 * (coalesce(max((data->'heap_blks_read')::int8) - min((data->'heap_blks_read')::int8), 0) + coalesce(max((data->'toast_blks_read')::int8) - min((data->'toast_blks_read')::int8), 0) + coalesce(max((data->'tidx_blks_read')::int8) - min((data->'tidx_blks_read')::int8), 0)) as blks_read\nFROM\n table_io_stats\nWHERE\n $__timeFilter(time)\n AND dbname = '$dbname'\nGROUP BY\n 1, 2\nHAVING\n max((data->'heap_blks_read')::int8) > min((data->'heap_blks_read')::int8)\nORDER BY\n 3 DESC NULLS LAST\nLIMIT\n $top\n) x\njoin lateral (select data from table_stats where dbname = x.dbname and tag_data->>'table_full_name' = x.table_full_name and $__timeFilter(time) order by time desc limit 1) y on true\n", + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "total_relation_size_b" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Most block IO reads", + "transform": "table", + "type": "table" + }, + { + "content": "Brought to you by: \"Cybertec", + "datasource": null, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 10, + "links": [], + "mode": "html", + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": false, + "rawSql": "SELECT\n $__time(time_column),\n value1\nFROM\n metric_table\nWHERE\n $__timeFilter(time_column)\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#56A64B", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": null, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "pluginVersion": "6.7.4", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false, + "ymax": null, + "ymin": null + }, + "tableColumn": "", + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\n0 as time,\ncoalesce ((\n\nselect 1 from (\nselect\n tag_data->>'table_full_name' as table_name,\n (data->'seq_scan')::int8 as seq_scan,\n lag((data->'seq_scan')::int8) over (partition by tag_data->>'table_full_name' order by time) as seq_scan_lag\nfrom\n table_stats\nwhere\n $__timeFilter(time)\n and not tag_data->>'schema' ~* 'temp'\n and tag_data->>'table_full_name' in (select tag_data->>'table_full_name' from table_stats where $__timeFilter(time) and dbname = '$dbname' limit 500)\n AND dbname = '$dbname'\n) x\nwhere seq_scan_lag > seq_scan\nlimit 1\n\n), 0) as was_reset", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": "1,1", + "title": "", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "", + "value": "0" + }, + { + "op": "=", + "text": "WARNING - stats reset detected for selected time range, results are unreliable", + "value": "1" + }, + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "first" + } + ], + "schemaVersion": 22, + "style": "dark", + "tags": [ + "pgwatch" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": null, + "definition": "", + "hide": 0, + "includeAll": false, + "index": -1, + "label": null, + "multi": false, + "name": "dbname", + "options": [], + "query": "SELECT DISTINCT dbname FROM admin.all_distinct_dbname_metrics WHERE metric = 'table_stats' ORDER BY 1;", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "tags": [], + "text": "5", + "value": "5" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "top", + "options": [ + { + "selected": false, + "text": "1", + "value": "1" + }, + { + "selected": false, + "text": "3", + "value": "3" + }, + { + "selected": true, + "text": "5", + "value": "5" + }, + { + "selected": false, + "text": "10", + "value": "10" + }, + { + "selected": false, + "text": "20", + "value": "20" + }, + { + "selected": false, + "text": "50", + "value": "50" + } + ], + "query": "1,3,5,10,20,50", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Tables Top", + "uid": "tables-top", + "version": 1 + } + diff --git a/helm/pgwatch/templates/pgwatch-deployment.yaml b/helm/pgwatch/templates/pgwatch-deployment.yaml new file mode 100644 index 0000000..155c96c --- /dev/null +++ b/helm/pgwatch/templates/pgwatch-deployment.yaml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + application: pgwatch + pgwatch.pods.role: pgwatch + name: pgwatch +spec: + replicas: 1 + selector: + matchLabels: + application: pgwatch + pgwatch.pods.role: pgwatch + template: + metadata: + labels: + application: pgwatch + pgwatch.pods.role: pgwatch + spec: + containers: + - env: + - name: PGWATCH_USER + valueFrom: + secretKeyRef: + name: pgwatch-postgresql-secret-pgwatch + key: username + - name: PGWATCH_USER_PASSWORD + valueFrom: + secretKeyRef: + name: pgwatch-postgresql-secret-pgwatch + key: password + image: {{ .Values.pgwatch.image }} + name: pgwatch + ports: + - containerPort: 8080 + protocol: TCP + command: ["/bin/sh", "-c"] + args: + - | + export PW_SINK=postgresql://$PGWATCH_USER:$PGWATCH_USER_PASSWORD@postgres-svc.pgwatch.svc.cluster.local:5432/pgwatch_metrics?sslmode=disable + export PW_SOURCES=postgresql://$PGWATCH_USER:$PGWATCH_USER_PASSWORD@postgres-svc.pgwatch.svc.cluster.local:5432/pgwatch?sslmode=disable + exec /pgwatch/pgwatch + restartPolicy: Always + diff --git a/helm/pgwatch/templates/postgres-statefulset.yaml b/helm/pgwatch/templates/postgres-statefulset.yaml new file mode 100644 index 0000000..b90ef36 --- /dev/null +++ b/helm/pgwatch/templates/postgres-statefulset.yaml @@ -0,0 +1,79 @@ + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres +spec: + serviceName: "postgres-svc" + replicas: 1 + selector: + matchLabels: + application: pgwatch + pgwatch.pods.role: postgres + template: + metadata: + labels: + application: pgwatch + pgwatch.pods.role: postgres + spec: + containers: + - name: postgres + image: {{ .Values.pgwatch.postgres.image }} + ports: + - containerPort: 5432 + name: postgres + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: pgwatch-postgresql-secret-postgres + key: password + - name: POSTGRES_USER + value: postgres + - name: ADDITIONAL_USER + valueFrom: + secretKeyRef: + name: pgwatch-postgresql-secret-pgwatch + key: username + - name: ADDITIONAL_USER_PASSWORD + valueFrom: + secretKeyRef: + name: pgwatch-postgresql-secret-pgwatch + key: password + - name: ADDITIONAL_DB1 + value: pgwatch + - name: ADDITIONAL_DB2 + value: pgwatch_metrics + - name: ADDITIONAL_DB3 + value: pgwatch_grafana + - name: PGDATA + value: /pgdata/data + volumeMounts: + - name: pgdata + mountPath: /pgdata + lifecycle: + postStart: + exec: + command: + - "/bin/bash" + - "-c" + - | + psql -U $POSTGRES_USER -c "DO \$\$ BEGIN IF EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '$ADDITIONAL_USER') THEN RAISE NOTICE 'Role \"$ADDITIONAL_USER\" already exists. Skipping.'; ELSE CREATE ROLE $ADDITIONAL_USER LOGIN PASSWORD '$ADDITIONAL_USER_PASSWORD'; END IF; END \$\$;" + psql -U $POSTGRES_USER -tc "SELECT 1 FROM pg_database WHERE datname = '$ADDITIONAL_DB1'" | grep -q 1 || psql -U $POSTGRES_USER -c "CREATE DATABASE $ADDITIONAL_DB1" + psql -U $POSTGRES_USER -tc "SELECT 1 FROM pg_database WHERE datname = '$ADDITIONAL_DB2'" | grep -q 1 || psql -U $POSTGRES_USER -c "CREATE DATABASE $ADDITIONAL_DB2" + psql -U $POSTGRES_USER -tc "SELECT 1 FROM pg_database WHERE datname = '$ADDITIONAL_DB3'" | grep -q 1 || psql -U $POSTGRES_USER -c "CREATE DATABASE $ADDITIONAL_DB3" + psql -U $POSTGRES_USER -c "ALTER DATABASE $ADDITIONAL_DB1 OWNER TO $ADDITIONAL_USER;" + psql -U $POSTGRES_USER -c "ALTER DATABASE $ADDITIONAL_DB2 OWNER TO $ADDITIONAL_USER;" + psql -U $POSTGRES_USER -c "ALTER DATABASE $ADDITIONAL_DB3 OWNER TO $ADDITIONAL_USER;" + volumeClaimTemplates: + - metadata: + name: pgdata + labels: + application: pgwatch + pgwatch.pods.role: postgres + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: {{ .Values.pgwatch.postgres.disk.storageClass }} + resources: + requests: + storage: {{ .Values.pgwatch.postgres.disk.size }} \ No newline at end of file diff --git a/helm/pgwatch/templates/secrets.yaml b/helm/pgwatch/templates/secrets.yaml new file mode 100644 index 0000000..b14c1ee --- /dev/null +++ b/helm/pgwatch/templates/secrets.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Secret +metadata: + name: pgwatch-postgresql-secret-postgres +type: Opaque +stringData: + password: 'beeF+u1bohce5xieZaamahChei3uthu>' + username: 'postgres' +--- + +apiVersion: v1 +kind: Secret +metadata: + name: pgwatch-postgresql-secret-pgwatch +type: Opaque +stringData: + password: 'awaes6ohR1dee2iedoo1n0Ao2lie3Le-' + username: 'pgwatch' +--- \ No newline at end of file diff --git a/helm/pgwatch/templates/services.yaml b/helm/pgwatch/templates/services.yaml new file mode 100644 index 0000000..8079b85 --- /dev/null +++ b/helm/pgwatch/templates/services.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + application: pgwatch + name: pgwatch-svc +spec: + ports: + - name: pgwatch + port: 8080 + targetPort: 8080 + selector: + application: pgwatch + pgwatch.pods.role: pgwatch +--- +apiVersion: v1 +kind: Service +metadata: + labels: + application: pgwatch + name: grafana-svc +spec: + ports: + - name: grafana + port: 3000 + targetPort: 3000 + selector: + application: pgwatch + pgwatch.pods.role: grafana +--- +apiVersion: v1 +kind: Service +metadata: + labels: + application: pgwatch + name: postgres-svc +spec: + ports: + - name: postgresql + port: 5432 + targetPort: 5432 + selector: + application: pgwatch + pgwatch.pods.role: postgres diff --git a/helm/pgwatch/values.yaml b/helm/pgwatch/values.yaml new file mode 100644 index 0000000..bf6ed03 --- /dev/null +++ b/helm/pgwatch/values.yaml @@ -0,0 +1,8 @@ +pgwatch: + image: "docker.io/cybertecpostgresql/pgwatch:latest" + postgres: + # image: "docker.io/bitnami/postgresql:latest" + image: "docker.io/schmaetz/postgres:bookworm-17-1" + disk: + size: '10Gi' + storageClass: 'crc-csi-hostpath-provisioner' diff --git a/helm/.helmignore b/helm/pgwatch2(old)/.helmignore similarity index 100% rename from helm/.helmignore rename to helm/pgwatch2(old)/.helmignore diff --git a/helm/Chart.lock b/helm/pgwatch2(old)/Chart.lock similarity index 100% rename from helm/Chart.lock rename to helm/pgwatch2(old)/Chart.lock diff --git a/helm/Chart.yaml b/helm/pgwatch2(old)/Chart.yaml similarity index 100% rename from helm/Chart.yaml rename to helm/pgwatch2(old)/Chart.yaml diff --git a/helm/README.md b/helm/pgwatch2(old)/README.md similarity index 100% rename from helm/README.md rename to helm/pgwatch2(old)/README.md diff --git a/helm/charts/influxdb-4.12.0.tgz b/helm/pgwatch2(old)/charts/influxdb-4.12.0.tgz similarity index 100% rename from helm/charts/influxdb-4.12.0.tgz rename to helm/pgwatch2(old)/charts/influxdb-4.12.0.tgz diff --git a/helm/charts/timescaledb-single-0.33.1.tgz b/helm/pgwatch2(old)/charts/timescaledb-single-0.33.1.tgz similarity index 100% rename from helm/charts/timescaledb-single-0.33.1.tgz rename to helm/pgwatch2(old)/charts/timescaledb-single-0.33.1.tgz diff --git a/helm/charts/grafana-6.50.0.tgz b/helm/pgwatch2(old)/grafana-6.50.0.tgz similarity index 100% rename from helm/charts/grafana-6.50.0.tgz rename to helm/pgwatch2(old)/grafana-6.50.0.tgz diff --git a/helm/grafana_dashboards b/helm/pgwatch2(old)/grafana_dashboards similarity index 100% rename from helm/grafana_dashboards rename to helm/pgwatch2(old)/grafana_dashboards diff --git a/helm/templates/NOTES.txt b/helm/pgwatch2(old)/templates/NOTES.txt similarity index 100% rename from helm/templates/NOTES.txt rename to helm/pgwatch2(old)/templates/NOTES.txt diff --git a/helm/templates/_helpers.tpl b/helm/pgwatch2(old)/templates/_helpers.tpl similarity index 100% rename from helm/templates/_helpers.tpl rename to helm/pgwatch2(old)/templates/_helpers.tpl diff --git a/helm/templates/configmaps.yaml b/helm/pgwatch2(old)/templates/configmaps.yaml similarity index 100% rename from helm/templates/configmaps.yaml rename to helm/pgwatch2(old)/templates/configmaps.yaml diff --git a/helm/templates/configmaps2.yaml b/helm/pgwatch2(old)/templates/configmaps2.yaml similarity index 100% rename from helm/templates/configmaps2.yaml rename to helm/pgwatch2(old)/templates/configmaps2.yaml diff --git a/helm/templates/deployment.yaml b/helm/pgwatch2(old)/templates/deployment.yaml similarity index 100% rename from helm/templates/deployment.yaml rename to helm/pgwatch2(old)/templates/deployment.yaml diff --git a/helm/templates/grafana-dashboards.yaml b/helm/pgwatch2(old)/templates/grafana-dashboards.yaml similarity index 100% rename from helm/templates/grafana-dashboards.yaml rename to helm/pgwatch2(old)/templates/grafana-dashboards.yaml diff --git a/helm/templates/ingress.yaml b/helm/pgwatch2(old)/templates/ingress.yaml similarity index 100% rename from helm/templates/ingress.yaml rename to helm/pgwatch2(old)/templates/ingress.yaml diff --git a/helm/templates/service.yaml b/helm/pgwatch2(old)/templates/service.yaml similarity index 100% rename from helm/templates/service.yaml rename to helm/pgwatch2(old)/templates/service.yaml diff --git a/helm/values.yaml b/helm/pgwatch2(old)/values.yaml similarity index 100% rename from helm/values.yaml rename to helm/pgwatch2(old)/values.yaml From 9df2f8e73fdfe484720e112f90c3f435dbd8d2bb Mon Sep 17 00:00:00 2001 From: matthias Date: Tue, 3 Dec 2024 13:08:22 +0100 Subject: [PATCH 02/32] added option for using an existing database as metric store --- .gitignore | 1 + helm/README.md | 161 ++++++++++++++++++ .../templates/grafana-dashboards-basics.yaml | 1 + .../pgwatch/templates/grafana-deployment.yaml | 34 +++- .../templates/grafana_dashboard_others.yaml | 1 + .../pgwatch/templates/pgwatch-deployment.yaml | 30 +++- .../templates/postgres-statefulset.yaml | 8 +- helm/pgwatch/templates/secrets.yaml | 6 +- helm/pgwatch/templates/services.yaml | 5 + helm/pgwatch/values.yaml | 22 ++- 10 files changed, 248 insertions(+), 21 deletions(-) create mode 100644 .gitignore create mode 100644 helm/README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b0cdd63 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +helm/pgwatch/test_values.yaml \ No newline at end of file diff --git a/helm/README.md b/helm/README.md new file mode 100644 index 0000000..1f2afbb --- /dev/null +++ b/helm/README.md @@ -0,0 +1,161 @@ +[![Documentation](https://img.shields.io/badge/Documentation-pgwat.ch-brightgreen)](https://pgwat.ch) +[![License: MIT](https://img.shields.io/badge/License-BSD_3-green.svg)](https://opensource.org/license/bsd-3-clause) +[![Go Build & Test](https://github.com/cybertec-postgresql/pgwatch/actions/workflows/build.yml/badge.svg)](https://github.com/cybertec-postgresql/pgwatch/actions/workflows/build.yml) +[![Coverage Status](https://coveralls.io/repos/github/cybertec-postgresql/pgwatch/badge.svg?branch=master&service=github)](https://coveralls.io/github/cybertec-postgresql/pgwatch?branch=master) + + +# pgwatch v3-beta. Please test it as much as possible! + +This is the next generation of [pgwatch2](https://github.com/cybertec-postgresql/pgwatch2/). + +## Quick Start + +To fetch and run the latest **demo** Docker image, exposing +- Grafana on port 3000, +- the administrative web UI on port 8080, +- the internal configuration and metrics database on port 5432: + +```shell +docker run -d --name pw3 -p 5432:5432 -p 3000:3000 -p 8080:8080 -e PW_TESTDB=true cybertecpostgresql/pgwatch-demo +``` + +After some minutes you could open the ["Database Overview"](http://127.0.0.1:3000/dashboard/db/db-overview) dashboard and start looking at metrics. For defining your own dashboards you need to log in Grafana as admin (`admin`/`pgwatchadmin`). + +If you don't want to add the test database for monitoring, remove the `PW_TESTDB` parameter when launching the container. + + + +## Development and production use + +For production and long term installation `cybertecpostgresql/pgwatch` Docker image should be used. +For the fastest development and deployment experience the Docker compose files are provided: + +```shell +git clone https://github.com/cybertec-postgresql/pgwatch.git && cd pgwatch + +docker compose -f ./docker/docker-compose.yml up --detach +``` +
+ ✔ Network pgwatch_default       Created
+ ✔ Container pgwatch-postgres-1  Healthy
+ ✔ Container pgwatch-pgwatch-1  Started
+ ✔ Container pgwatch-grafana-1   Started
+
+ +These commands will build and start services listed in the compose file: +- configuration and metric database; +- pgwatch monitoring agent with WebUI; +- Grafana with dashboards. + +## Monitor Database + +After start, you could open the [monitoring dashboard](http://localhost:3000/) and start +looking at metrics. + +To add a test database under monitoring, you can use [built-in WebUI](http://localhost:8080/). Or simply +execute from command line: +```shell +docker compose -f ./docker/docker-compose.yml up add-test-db --force-recreate +``` +
+[+] Running 2/0
+ ✔ Container pgwatch-postgres-1     Running                                                                       0.0s
+ ✔ Container pgwatch-add-test-db-1  Created                                                                       0.0s
+Attaching to pgwatch-add-test-db-1
+pgwatch-add-test-db-1  | BEGIN
+...
+pgwatch-add-test-db-1  | GRANT
+pgwatch-add-test-db-1  | COMMENT
+pgwatch-add-test-db-1  | INSERT 0 1
+pgwatch-add-test-db-1 exited with code 0
+
+ +## Produce Workload + +To emulate workload for added test database execute: +```shell +docker compose -f ./docker/docker-compose.yml up pgbench +``` +
+[+] Running 2/2
+ ✔ Container pgwatch-postgres-1  Running                                                                          0.0s
+ ✔ Container pgwatch-pgbench-1   Created                                                                          0.1s
+Attaching to pgwatch-pgbench-1
+pgwatch-pgbench-1  | dropping old tables...
+pgwatch-pgbench-1  | NOTICE:  table "pgbench_accounts" does not exist, skipping
+pgwatch-pgbench-1  | NOTICE:  table "pgbench_branches" does not exist, skipping
+pgwatch-pgbench-1  | NOTICE:  table "pgbench_history" does not exist, skipping
+pgwatch-pgbench-1  | NOTICE:  table "pgbench_tellers" does not exist, skipping
+pgwatch-pgbench-1  | creating tables...
+pgwatch-pgbench-1  | generating data (client-side)...
+pgwatch-pgbench-1  | 100000 of 5000000 tuples (2%) done (elapsed 0.11 s, remaining 5.17 s)
+pgwatch-pgbench-1  | 200000 of 5000000 tuples (4%) done (elapsed 0.25 s, remaining 6.06 s)
+...
+pgwatch-pgbench-1  | 5000000 of 5000000 tuples (100%) done (elapsed 16.28 s, remaining 0.00 s)
+pgwatch-pgbench-1  | vacuuming...
+pgwatch-pgbench-1  | creating primary keys...
+pgwatch-pgbench-1  | done in 42.29 s (drop tables 0.03 s, create tables 0.04 s, client-side generate 18.23 s, vacuum 1.29 s, primary keys 22.70 s).
+pgwatch-pgbench-1  | pgbench (15.4)
+pgwatch-pgbench-1  | starting vacuum...
+pgwatch-pgbench-1  | end.
+pgwatch-pgbench-1  | progress: 5.0 s, 642.2 tps, lat 15.407 ms stddev 11.794, 0 failed
+pgwatch-pgbench-1  | progress: 10.0 s, 509.6 tps, lat 19.541 ms stddev 9.493, 0 failed
+...
+pgwatch-pgbench-1  | progress: 185.0 s, 325.3 tps, lat 16.825 ms stddev 8.330, 0 failed
+pgwatch-pgbench-1  |
+pgwatch-pgbench-1  |
+pgwatch-pgbench-1  | transaction type: builtin: TPC-B (sort of)
+pgwatch-pgbench-1  | scaling factor: 50
+pgwatch-pgbench-1  | query mode: simple
+pgwatch-pgbench-1  | number of clients: 10
+pgwatch-pgbench-1  | number of threads: 2
+pgwatch-pgbench-1  | maximum number of tries: 1
+pgwatch-pgbench-1  | number of transactions per client: 10000
+pgwatch-pgbench-1  | number of transactions actually processed: 100000/100000
+pgwatch-pgbench-1  | number of failed transactions: 0 (0.000%)
+pgwatch-pgbench-1  | latency average = 18.152 ms
+pgwatch-pgbench-1  | latency stddev = 13.732 ms
+pgwatch-pgbench-1  | initial connection time = 25.085 ms
+pgwatch-pgbench-1  | tps = 534.261013 (without initial connection time)
+pgwatch-pgbench-1  | dropping old tables...
+pgwatch-pgbench-1  | done in 0.45 s (drop tables 0.45 s).
+pgwatch-pgbench-1 exited with code 0
+
+ +## Inspect database + +> [!IMPORTANT] +pgAdmin uses port 80. If you want it to use another port, change it in `docker-compose.yml` file. + +To look what is inside `pgwatch` database, you can spin up pgAdmin4: +```shell +docker compose -f ./docker/docker-compose.yml up --detach pgadmin +``` +Go to `localhost` in your favorite browser and login as `admin@local.com`, password `admin`. +Server `pgwatch` should be already added in `Servers` group. + +## Development + +If you apply any changes to the source code and want to restart the agent, it's usually enough to run: + +```shell +docker compose -f ./docker/docker-compose.yml up pgwatch --build --force-recreate --detach +``` + +The command above will rebuild the `pgwatch` agent from sources and relaunch the container. + +## Logs + +If you are running containers in detached mode, you still can follow the logs: +```shell +docker compose -f ./docker/docker-compose.yml logs --follow +``` + +Or you may check the log of a particular service: +```shell +docker compose -f ./docker/docker-compose.yml logs pgwatch --follow +``` + +# Contributing + +Feedback, suggestions, problem reports, and pull requests are very much appreciated. diff --git a/helm/pgwatch/templates/grafana-dashboards-basics.yaml b/helm/pgwatch/templates/grafana-dashboards-basics.yaml index 4bd4329..c601102 100644 --- a/helm/pgwatch/templates/grafana-dashboards-basics.yaml +++ b/helm/pgwatch/templates/grafana-dashboards-basics.yaml @@ -4,6 +4,7 @@ metadata: labels: application: pgwatch name: grafana-dashboards-basics + namespace: {{ .Release.Namespace }} data: db-overview-developer.json: |- { diff --git a/helm/pgwatch/templates/grafana-deployment.yaml b/helm/pgwatch/templates/grafana-deployment.yaml index 05acc48..aa70ecb 100644 --- a/helm/pgwatch/templates/grafana-deployment.yaml +++ b/helm/pgwatch/templates/grafana-deployment.yaml @@ -5,6 +5,7 @@ metadata: application: pgwatch pgwatch.pods.role: grafana name: grafana + namespace: {{ .Release.Namespace }} spec: replicas: 1 selector: @@ -27,26 +28,39 @@ spec: value: Admin - name: GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH value: /var/lib/grafana/dashboards/1-global-db-overview.json + - name: GF_DATABASE_TYPE + value: postgres + - name: GF_INSTALL_PLUGINS + value: marcusolsson-treemap-panel + {{- if eq .Values.pgwatch.postgres.create_metric_database "true" }} - name: GF_DATABASE_HOST value: postgres-svc:5432 - name: GF_DATABASE_NAME value: pgwatch_grafana - - name: GF_DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: pgwatch-postgresql-secret-pgwatch - key: password - name: GF_DATABASE_SSL_MODE value: disable - - name: GF_DATABASE_TYPE - value: postgres - name: GF_DATABASE_USER valueFrom: secretKeyRef: name: pgwatch-postgresql-secret-pgwatch key: username - - name: GF_INSTALL_PLUGINS - value: marcusolsson-treemap-panel + - name: GF_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: pgwatch-postgresql-secret-pgwatch + key: password + {{- else }} + - name: GF_DATABASE_HOST + value: {{ .Values.pgwatch.postgres.use_existing_database.endpoint }} + - name: GF_DATABASE_NAME + value: {{ .Values.pgwatch.postgres.use_existing_database.database }} + - name: GF_DATABASE_SSL_MODE + value: {{ .Values.pgwatch.postgres.use_existing_database.sslmode }} + - name: GF_DATABASE_USER + value: {{ .Values.pgwatch.postgres.use_existing_database.username }} + - name: GF_DATABASE_PASSWORD + value: {{ .Values.pgwatch.postgres.use_existing_database.password }} + {{ end }} image: grafana/grafana:10.4.7 name: grafana ports: @@ -94,6 +108,7 @@ metadata: labels: application: pgwatch name: grafana-datasources + namespace: {{ .Release.Namespace }} data: postgres_datasource.yml: "apiVersion: 1\n\ndatasources:\n- name: pg-metrics\n type: postgres\n url: postgres-svc:5432\n access: proxy\n password: pgwatchadmin\n user: pgwatch\n database: pgwatch_metrics\n basicAuth: false\n isDefault: true\n jsonData:\n sslmode: disable\n postgresVersion: 1700 \n version: 1\n editable: true\n" --- @@ -103,6 +118,7 @@ metadata: labels: application: pgwatch name: grafana-config + namespace: {{ .Release.Namespace }} data: postgres_dashboard.yml: | apiVersion: 1 diff --git a/helm/pgwatch/templates/grafana_dashboard_others.yaml b/helm/pgwatch/templates/grafana_dashboard_others.yaml index 24f3f78..e6daf82 100644 --- a/helm/pgwatch/templates/grafana_dashboard_others.yaml +++ b/helm/pgwatch/templates/grafana_dashboard_others.yaml @@ -4,6 +4,7 @@ metadata: labels: application: pgwatch name: grafana-dashboards-others + namespace: {{ .Values.pgwatch.namespace }} data: 1-global-db-overview.json: |- { diff --git a/helm/pgwatch/templates/pgwatch-deployment.yaml b/helm/pgwatch/templates/pgwatch-deployment.yaml index 155c96c..bc3c1fe 100644 --- a/helm/pgwatch/templates/pgwatch-deployment.yaml +++ b/helm/pgwatch/templates/pgwatch-deployment.yaml @@ -5,6 +5,7 @@ metadata: application: pgwatch pgwatch.pods.role: pgwatch name: pgwatch + namespace: {{ .Release.Namespace }} spec: replicas: 1 selector: @@ -18,7 +19,8 @@ spec: pgwatch.pods.role: pgwatch spec: containers: - - env: + - env: + {{- if eq .Values.pgwatch.postgres.create_metric_database "true" }} - name: PGWATCH_USER valueFrom: secretKeyRef: @@ -29,6 +31,28 @@ spec: secretKeyRef: name: pgwatch-postgresql-secret-pgwatch key: password + - name: METRIC_DATABASE_ENDPOINT + value: "postgres-svc.{{ .Release.Namespace }}.svc.cluster.local" + - name: METRIC_DATABASE_PORT + value: "5432" + - name: METRIC_DATABASE_DATABASE + value: pgwatch_metrics + - name: METRIC_DATABASE_SSLMODE + value: disable + {{- else }} + - name: PGWATCH_USER + value: {{ .Values.pgwatch.postgres.use_existing_database.username }} + - name: PGWATCH_USER_PASSWORD + value: {{ .Values.pgwatch.postgres.use_existing_database.password }} + - name: METRIC_DATABASE_ENDPOINT + value: {{ .Values.pgwatch.postgres.use_existing_database.endpoint }} + - name: METRIC_DATABASE_PORT + value: "{{ .Values.pgwatch.postgres.use_existing_database.port }}" + - name: METRIC_DATABASE_DATABASE + value: {{ .Values.pgwatch.postgres.use_existing_database.database }} + - name: METRIC_DATABASE_SSLMODE + value: {{ .Values.pgwatch.postgres.use_existing_database.sslmode }} + {{ end }} image: {{ .Values.pgwatch.image }} name: pgwatch ports: @@ -37,8 +61,8 @@ spec: command: ["/bin/sh", "-c"] args: - | - export PW_SINK=postgresql://$PGWATCH_USER:$PGWATCH_USER_PASSWORD@postgres-svc.pgwatch.svc.cluster.local:5432/pgwatch_metrics?sslmode=disable - export PW_SOURCES=postgresql://$PGWATCH_USER:$PGWATCH_USER_PASSWORD@postgres-svc.pgwatch.svc.cluster.local:5432/pgwatch?sslmode=disable + export PW_SINK=postgresql://$PGWATCH_USER:$PGWATCH_USER_PASSWORD@$METRIC_DATABASE_ENDPOINT:$METRIC_DATABASE_PORT/$METRIC_DATABASE_DATABASE?sslmode=$METRIC_DATABASE_SSLMODE + export PW_SOURCES=postgresql://$PGWATCH_USER:$PGWATCH_USER_PASSWORD@$METRIC_DATABASE_ENDPOINT:$METRIC_DATABASE_PORT/$METRIC_DATABASE_DATABASE?sslmode=$METRIC_DATABASE_SSLMODE exec /pgwatch/pgwatch restartPolicy: Always diff --git a/helm/pgwatch/templates/postgres-statefulset.yaml b/helm/pgwatch/templates/postgres-statefulset.yaml index b90ef36..865a2ca 100644 --- a/helm/pgwatch/templates/postgres-statefulset.yaml +++ b/helm/pgwatch/templates/postgres-statefulset.yaml @@ -1,8 +1,9 @@ - +{{- if eq .Values.pgwatch.postgres.create_metric_database "true" }} apiVersion: apps/v1 kind: StatefulSet metadata: name: postgres + namespace: {{ .Release.Namespace }} spec: serviceName: "postgres-svc" replicas: 1 @@ -73,7 +74,8 @@ spec: pgwatch.pods.role: postgres spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.pgwatch.postgres.disk.storageClass }} + storageClassName: {{ .Values.pgwatch.postgres.new_metric_database.disk.storageClass }} resources: requests: - storage: {{ .Values.pgwatch.postgres.disk.size }} \ No newline at end of file + storage: {{ .Values.pgwatch.postgres.new_metric_database.disk.size }} +{{ end }} \ No newline at end of file diff --git a/helm/pgwatch/templates/secrets.yaml b/helm/pgwatch/templates/secrets.yaml index b14c1ee..38928fa 100644 --- a/helm/pgwatch/templates/secrets.yaml +++ b/helm/pgwatch/templates/secrets.yaml @@ -1,7 +1,9 @@ +{{- if eq .Values.pgwatch.postgres.create_metric_database "true" }} apiVersion: v1 kind: Secret metadata: name: pgwatch-postgresql-secret-postgres + namespace: {{ .Release.Namespace }} type: Opaque stringData: password: 'beeF+u1bohce5xieZaamahChei3uthu>' @@ -12,8 +14,10 @@ apiVersion: v1 kind: Secret metadata: name: pgwatch-postgresql-secret-pgwatch + namespace: {{ .Release.Namespace }} type: Opaque stringData: password: 'awaes6ohR1dee2iedoo1n0Ao2lie3Le-' username: 'pgwatch' ---- \ No newline at end of file +--- +{{ end }} \ No newline at end of file diff --git a/helm/pgwatch/templates/services.yaml b/helm/pgwatch/templates/services.yaml index 8079b85..1305be9 100644 --- a/helm/pgwatch/templates/services.yaml +++ b/helm/pgwatch/templates/services.yaml @@ -4,6 +4,7 @@ metadata: labels: application: pgwatch name: pgwatch-svc + namespace: {{ .Release.Namespace }} spec: ports: - name: pgwatch @@ -19,6 +20,7 @@ metadata: labels: application: pgwatch name: grafana-svc + namespace: {{ .Release.Namespace }} spec: ports: - name: grafana @@ -28,12 +30,14 @@ spec: application: pgwatch pgwatch.pods.role: grafana --- +{{- if eq .Values.pgwatch.postgres.create_metric_database "true" }} apiVersion: v1 kind: Service metadata: labels: application: pgwatch name: postgres-svc + namespace: {{ .Release.Namespace }} spec: ports: - name: postgresql @@ -42,3 +46,4 @@ spec: selector: application: pgwatch pgwatch.pods.role: postgres +{{ end }} \ No newline at end of file diff --git a/helm/pgwatch/values.yaml b/helm/pgwatch/values.yaml index bf6ed03..9d6fe2d 100644 --- a/helm/pgwatch/values.yaml +++ b/helm/pgwatch/values.yaml @@ -1,8 +1,20 @@ pgwatch: image: "docker.io/cybertecpostgresql/pgwatch:latest" + namespace: pgwatch postgres: - # image: "docker.io/bitnami/postgresql:latest" - image: "docker.io/schmaetz/postgres:bookworm-17-1" - disk: - size: '10Gi' - storageClass: 'crc-csi-hostpath-provisioner' + # define is a database for the metrics needs to be created or if there is already an existing database + create_metric_database: "true" #"false" + new_metric_database: + image: "docker.io/schmaetz/postgres:bookworm-17-1" + disk: + size: '10Gi' + storageClass: 'crc-csi-hostpath-provisioner' + use_existing_database: + endpoint: postgresql.local + port: 5432 + database: PGWATCH_DATABASE + sslmode: disable + username: pgwatch_user + password: PASSWORD_FOR_PGWATCH_USER + + From 90a7ebc660c4135664bd5a6ae548bb2da2761275 Mon Sep 17 00:00:00 2001 From: matthias Date: Tue, 3 Dec 2024 13:15:27 +0100 Subject: [PATCH 03/32] change default values --- helm/pgwatch/templates/postgres-statefulset.yaml | 2 +- helm/pgwatch/values.yaml | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/helm/pgwatch/templates/postgres-statefulset.yaml b/helm/pgwatch/templates/postgres-statefulset.yaml index 865a2ca..180f809 100644 --- a/helm/pgwatch/templates/postgres-statefulset.yaml +++ b/helm/pgwatch/templates/postgres-statefulset.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: postgres - image: {{ .Values.pgwatch.postgres.image }} + image: {{ .Values.pgwatch.postgres.new_metric_database.image }} ports: - containerPort: 5432 name: postgres diff --git a/helm/pgwatch/values.yaml b/helm/pgwatch/values.yaml index 9d6fe2d..68bc9b2 100644 --- a/helm/pgwatch/values.yaml +++ b/helm/pgwatch/values.yaml @@ -9,12 +9,12 @@ pgwatch: disk: size: '10Gi' storageClass: 'crc-csi-hostpath-provisioner' - use_existing_database: - endpoint: postgresql.local - port: 5432 - database: PGWATCH_DATABASE - sslmode: disable - username: pgwatch_user - password: PASSWORD_FOR_PGWATCH_USER + # use_existing_database: + # endpoint: postgresql.local + # port: '5432' + # database: PGWATCH_DATABASE + # sslmode: disable + # username: pgwatch_user + # password: PASSWORD_FOR_PGWATCH_USER From 7c5595c2423eff9f0d9807715ffcba97b1f975bc Mon Sep 17 00:00:00 2001 From: matthias Date: Tue, 3 Dec 2024 13:15:51 +0100 Subject: [PATCH 04/32] update Helm-Chart-Version --- helm/pgwatch/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/pgwatch/Chart.yaml b/helm/pgwatch/Chart.yaml index 9faa66b..957c94d 100644 --- a/helm/pgwatch/Chart.yaml +++ b/helm/pgwatch/Chart.yaml @@ -2,4 +2,4 @@ apiVersion: v1 appVersion: "3.0" description: A Helm chart for pgwatch monitoring tools name: pgwatch -version: 3.0.0 +version: 3.0.1 From abd8207b24634cdc87bb04a6d30f1e8b421d52a3 Mon Sep 17 00:00:00 2001 From: matthias Date: Fri, 4 Apr 2025 13:50:56 +0200 Subject: [PATCH 05/32] added option for prometheus sink and allows to bring your own prom or create it via helm-chart --- .../templates/grafana_dashboard_others.yaml | 2 +- .../pgwatch/templates/pgwatch-deployment.yaml | 5 ++ .../templates/postgres-statefulset.yaml | 4 +- helm/pgwatch/templates/prometheus-config.yaml | 19 +++++ .../templates/prometheus-deployment.yaml | 81 +++++++++++++++++++ helm/pgwatch/templates/prometheus_rbac.yaml | 47 +++++++++++ helm/pgwatch/templates/pvcs.yaml | 17 ++++ helm/pgwatch/templates/services.yaml | 21 +++++ helm/pgwatch/values.yaml | 12 ++- 9 files changed, 202 insertions(+), 6 deletions(-) create mode 100644 helm/pgwatch/templates/prometheus-config.yaml create mode 100644 helm/pgwatch/templates/prometheus-deployment.yaml create mode 100644 helm/pgwatch/templates/prometheus_rbac.yaml create mode 100644 helm/pgwatch/templates/pvcs.yaml diff --git a/helm/pgwatch/templates/grafana_dashboard_others.yaml b/helm/pgwatch/templates/grafana_dashboard_others.yaml index e6daf82..ed9e078 100644 --- a/helm/pgwatch/templates/grafana_dashboard_others.yaml +++ b/helm/pgwatch/templates/grafana_dashboard_others.yaml @@ -4,7 +4,7 @@ metadata: labels: application: pgwatch name: grafana-dashboards-others - namespace: {{ .Values.pgwatch.namespace }} + namespace: {{ .Release.Namespace }} data: 1-global-db-overview.json: |- { diff --git a/helm/pgwatch/templates/pgwatch-deployment.yaml b/helm/pgwatch/templates/pgwatch-deployment.yaml index bc3c1fe..f629e71 100644 --- a/helm/pgwatch/templates/pgwatch-deployment.yaml +++ b/helm/pgwatch/templates/pgwatch-deployment.yaml @@ -58,11 +58,16 @@ spec: ports: - containerPort: 8080 protocol: TCP + {{- if eq .Values.pgwatch.prometheus.enable_prom_sink "true" }} + - containerPort: 9188 + protocol: TCP + {{ end }} command: ["/bin/sh", "-c"] args: - | export PW_SINK=postgresql://$PGWATCH_USER:$PGWATCH_USER_PASSWORD@$METRIC_DATABASE_ENDPOINT:$METRIC_DATABASE_PORT/$METRIC_DATABASE_DATABASE?sslmode=$METRIC_DATABASE_SSLMODE export PW_SOURCES=postgresql://$PGWATCH_USER:$PGWATCH_USER_PASSWORD@$METRIC_DATABASE_ENDPOINT:$METRIC_DATABASE_PORT/$METRIC_DATABASE_DATABASE?sslmode=$METRIC_DATABASE_SSLMODE exec /pgwatch/pgwatch + {{- if eq .Values.pgwatch.prometheus.enable_prom_sink "true" }} --sink=prometheus://0.0.0.0:9188/pgwatch {{- end }} \ restartPolicy: Always diff --git a/helm/pgwatch/templates/postgres-statefulset.yaml b/helm/pgwatch/templates/postgres-statefulset.yaml index 180f809..2a26a84 100644 --- a/helm/pgwatch/templates/postgres-statefulset.yaml +++ b/helm/pgwatch/templates/postgres-statefulset.yaml @@ -74,8 +74,8 @@ spec: pgwatch.pods.role: postgres spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.pgwatch.postgres.new_metric_database.disk.storageClass }} + storageClassName: {{ .Values.pgwatch.postgres.new_metric_database.volume.storageClass }} resources: requests: - storage: {{ .Values.pgwatch.postgres.new_metric_database.disk.size }} + storage: {{ .Values.pgwatch.postgres.new_metric_database.volume.size }} {{ end }} \ No newline at end of file diff --git a/helm/pgwatch/templates/prometheus-config.yaml b/helm/pgwatch/templates/prometheus-config.yaml new file mode 100644 index 0000000..d023553 --- /dev/null +++ b/helm/pgwatch/templates/prometheus-config.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/name: postgres-operator-monitoring + vendor: opensource.cybertec + name: pgwatch-prometheus-cm +data: + prometheus.yml: | + --- + global: + scrape_interval: 1m + scrape_timeout: 15s + evaluation_interval: 5s + + scrape_configs: + - job_name: 'pgwatch' + static_configs: + - targets: ['pgwatch-svc.pgwatch.svc.cluster.local:9188'] diff --git a/helm/pgwatch/templates/prometheus-deployment.yaml b/helm/pgwatch/templates/prometheus-deployment.yaml new file mode 100644 index 0000000..dc59ecf --- /dev/null +++ b/helm/pgwatch/templates/prometheus-deployment.yaml @@ -0,0 +1,81 @@ +{{- if eq .Values.pgwatch.prometheus.new_prometheus.create_prometheus "true" }} +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: pgwatch + vendor: opensource.cybertec + postgres-operator.cybertec.at/stack: pgwatch + name: pgwatch-prometheus +spec: + selector: + matchLabels: + postgres-operator.cybertec.at/stack: pgwatch + name: pgwatch-prometheus + template: + metadata: + creationTimestamp: null + labels: + postgres-operator.cybertec.at/stack: pgwatch + name: pgwatch-prometheus + spec: + containers: + - image: {{ .Values.pgwatch.prometheus.new_prometheus.image }} + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /-/healthy + port: 9090 + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 20 + successThreshold: 1 + timeoutSeconds: 1 + name: pgwatch-prometheus + ports: + - containerPort: 9090 + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /-/ready + port: 9090 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/prometheus + name: prometheus-cm + - mountPath: /prometheus + name: prometheus-volume + securityContext: {} + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + terminationGracePeriodSeconds: 30 + volumes: + - configMap: + defaultMode: 420 + name: pgwatch-prometheus-cm + name: prometheus-cm + - name: prometheus-volume + persistentVolumeClaim: + claimName: prometheus-pvc + - configMap: + defaultMode: 420 + name: alertmanager-rules-config + name: alertmanagerrules + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + +{{ end }} \ No newline at end of file diff --git a/helm/pgwatch/templates/prometheus_rbac.yaml b/helm/pgwatch/templates/prometheus_rbac.yaml new file mode 100644 index 0000000..306e4e9 --- /dev/null +++ b/helm/pgwatch/templates/prometheus_rbac.yaml @@ -0,0 +1,47 @@ +{{- if eq .Values.pgwatch.prometheus.new_prometheus.create_prometheus "true" }} +# apiVersion: v1 +# kind: ServiceAccount +# metadata: +# labels: +# app.kubernetes.io/name: cpo-monitoring +# vendor: opensource.cybertec +# name: pgwatch-prometheus + +# --- + +# apiVersion: rbac.authorization.k8s.io/v1 +# kind: ClusterRole +# metadata: +# labels: +# app.kubernetes.io/name: cpo-monitoring +# vendor: opensource.cybertec +# name: pgwatch-prometheus +# rules: +# - resources: +# - pods +# apiGroups: +# - "" +# verbs: +# - get +# - list +# - watch + +# --- + +# apiVersion: rbac.authorization.k8s.io/v1 +# kind: ClusterRoleBinding +# metadata: +# labels: +# app.kubernetes.io/name: cpo-monitoring +# vendor: opensource.cybertec +# name: pgwatch-prometheus +# roleRef: +# apiGroup: rbac.authorization.k8s.io +# kind: ClusterRole +# name: pgwatch-prometheus +# subjects: +# - kind: ServiceAccount +# name: pgwatch-prometheus +# namespace: cpo-monitoring + +{{ end }} diff --git a/helm/pgwatch/templates/pvcs.yaml b/helm/pgwatch/templates/pvcs.yaml new file mode 100644 index 0000000..92b738f --- /dev/null +++ b/helm/pgwatch/templates/pvcs.yaml @@ -0,0 +1,17 @@ +{{- if eq .Values.pgwatch.prometheus.new_prometheus.create_prometheus "true" }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/name: cpo-monitoring + vendor: opensource.cybertec + name: prometheus-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.pgwatch.prometheus.new_prometheus.volume.size }} + storageClassName: {{ .Values.pgwatch.prometheus.new_prometheus.volume.storageClass }} +{{ end }} \ No newline at end of file diff --git a/helm/pgwatch/templates/services.yaml b/helm/pgwatch/templates/services.yaml index 1305be9..7624232 100644 --- a/helm/pgwatch/templates/services.yaml +++ b/helm/pgwatch/templates/services.yaml @@ -10,6 +10,11 @@ spec: - name: pgwatch port: 8080 targetPort: 8080 +{{- if eq .Values.pgwatch.prometheus.new_prometheus.create_prometheus "true" }} + - name: pgwatch-prometheus-sink + port: 9188 + targetPort: 9188 +{{ end }} selector: application: pgwatch pgwatch.pods.role: pgwatch @@ -46,4 +51,20 @@ spec: selector: application: pgwatch pgwatch.pods.role: postgres +{{ end }} +--- +{{- if eq .Values.pgwatch.prometheus.new_prometheus.create_prometheus "true" }} +apiVersion: v1 +kind: Service +metadata: + labels: + application: pgwatch + name: pgwatch-prometheus-svc +spec: + type: ClusterIP + ports: + - name: prometheus + port: 9090 + selector: + name: pgwatch-prometheus {{ end }} \ No newline at end of file diff --git a/helm/pgwatch/values.yaml b/helm/pgwatch/values.yaml index 68bc9b2..31b3389 100644 --- a/helm/pgwatch/values.yaml +++ b/helm/pgwatch/values.yaml @@ -6,7 +6,7 @@ pgwatch: create_metric_database: "true" #"false" new_metric_database: image: "docker.io/schmaetz/postgres:bookworm-17-1" - disk: + volume: size: '10Gi' storageClass: 'crc-csi-hostpath-provisioner' # use_existing_database: @@ -16,5 +16,11 @@ pgwatch: # sslmode: disable # username: pgwatch_user # password: PASSWORD_FOR_PGWATCH_USER - - + prometheus: + enable_prom_sink: "true" + new_prometheus: + create_prometheus: "true" + image: "prom/prometheus:main" + volume: + size: '10Gi' + storageClass: 'crc-csi-hostpath-provisioner' From ff998dd8f64a1adb8cd45e3029d59dcb436f6ee9 Mon Sep 17 00:00:00 2001 From: matthias Date: Fri, 4 Apr 2025 14:50:39 +0200 Subject: [PATCH 06/32] replace cmd modifications by mounting launcher script --- helm/pgwatch/templates/pgwatch-config.yaml | 23 +++++++++++++++++ .../pgwatch/templates/pgwatch-deployment.yaml | 25 +++++++++++++------ .../templates/prometheus-deployment.yaml | 4 --- helm/pgwatch/values.yaml | 1 + 4 files changed, 42 insertions(+), 11 deletions(-) create mode 100644 helm/pgwatch/templates/pgwatch-config.yaml diff --git a/helm/pgwatch/templates/pgwatch-config.yaml b/helm/pgwatch/templates/pgwatch-config.yaml new file mode 100644 index 0000000..1580de4 --- /dev/null +++ b/helm/pgwatch/templates/pgwatch-config.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/name: postgres-operator-monitoring + vendor: opensource.cybertec + name: pgwatch-cm +data: + launcher.sh: | + #!/bin/sh + + CMD="/pgwatch/pgwatch --sources=postgresql://$PGWATCH_USER:$PGWATCH_USER_PASSWORD@$METRIC_DATABASE_ENDPOINT:$METRIC_DATABASE_PORT/$METRIC_DATABASE_DATABASE?sslmode=$METRIC_DATABASE_SSLMODE" + + if [[ "$PG_IS_SINK" == "true" ]]; then + CMD="$CMD --sink=postgresql://$PGWATCH_USER:$PGWATCH_USER_PASSWORD@$METRIC_DATABASE_ENDPOINT:$METRIC_DATABASE_PORT/$METRIC_DATABASE_DATABASE?sslmode=$METRIC_DATABASE_SSLMODE" + fi + + if [[ "$PROM_IS_SINK" == "true" ]]; then + CMD="$CMD --sink=prometheus://0.0.0.0:9188/pgwatch" + fi + + exec $CMD + diff --git a/helm/pgwatch/templates/pgwatch-deployment.yaml b/helm/pgwatch/templates/pgwatch-deployment.yaml index f629e71..27077aa 100644 --- a/helm/pgwatch/templates/pgwatch-deployment.yaml +++ b/helm/pgwatch/templates/pgwatch-deployment.yaml @@ -52,8 +52,20 @@ spec: value: {{ .Values.pgwatch.postgres.use_existing_database.database }} - name: METRIC_DATABASE_SSLMODE value: {{ .Values.pgwatch.postgres.use_existing_database.sslmode }} + {{ end }} + {{- if eq .Values.pgwatch.postgres.enable_pg_sink "true" }} + - name: PG_IS_SINK + value: "true" + {{ end }} + {{- if eq .Values.pgwatch.prometheus.enable_prom_sink "true" }} + - name: PROM_IS_SINK + value: "true" {{ end }} image: {{ .Values.pgwatch.image }} + volumeMounts: + - name: launcher + mountPath: /tmp/launcher.sh + subPath: launcher.sh name: pgwatch ports: - containerPort: 8080 @@ -62,12 +74,11 @@ spec: - containerPort: 9188 protocol: TCP {{ end }} - command: ["/bin/sh", "-c"] - args: - - | - export PW_SINK=postgresql://$PGWATCH_USER:$PGWATCH_USER_PASSWORD@$METRIC_DATABASE_ENDPOINT:$METRIC_DATABASE_PORT/$METRIC_DATABASE_DATABASE?sslmode=$METRIC_DATABASE_SSLMODE - export PW_SOURCES=postgresql://$PGWATCH_USER:$PGWATCH_USER_PASSWORD@$METRIC_DATABASE_ENDPOINT:$METRIC_DATABASE_PORT/$METRIC_DATABASE_DATABASE?sslmode=$METRIC_DATABASE_SSLMODE - exec /pgwatch/pgwatch - {{- if eq .Values.pgwatch.prometheus.enable_prom_sink "true" }} --sink=prometheus://0.0.0.0:9188/pgwatch {{- end }} \ + command: ["/tmp/launcher.sh"] + volumes: + - name: launcher + configMap: + name: pgwatch-cm + defaultMode: 0777 # Damit es direkt ausführbar ist restartPolicy: Always diff --git a/helm/pgwatch/templates/prometheus-deployment.yaml b/helm/pgwatch/templates/prometheus-deployment.yaml index dc59ecf..4bb45dd 100644 --- a/helm/pgwatch/templates/prometheus-deployment.yaml +++ b/helm/pgwatch/templates/prometheus-deployment.yaml @@ -65,10 +65,6 @@ spec: - name: prometheus-volume persistentVolumeClaim: claimName: prometheus-pvc - - configMap: - defaultMode: 420 - name: alertmanager-rules-config - name: alertmanagerrules progressDeadlineSeconds: 600 replicas: 1 revisionHistoryLimit: 10 diff --git a/helm/pgwatch/values.yaml b/helm/pgwatch/values.yaml index 31b3389..d3f2bc4 100644 --- a/helm/pgwatch/values.yaml +++ b/helm/pgwatch/values.yaml @@ -2,6 +2,7 @@ pgwatch: image: "docker.io/cybertecpostgresql/pgwatch:latest" namespace: pgwatch postgres: + enable_pg_sink: "true" # define is a database for the metrics needs to be created or if there is already an existing database create_metric_database: "true" #"false" new_metric_database: From 2e12348563d73401f4922cd352185e99d03d2235 Mon Sep 17 00:00:00 2001 From: matthias Date: Mon, 7 Apr 2025 08:37:19 +0200 Subject: [PATCH 07/32] added retention setting for pg sink --- helm/pgwatch/templates/pgwatch-config.yaml | 2 +- helm/pgwatch/templates/pgwatch-deployment.yaml | 2 ++ helm/pgwatch/values.yaml | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/helm/pgwatch/templates/pgwatch-config.yaml b/helm/pgwatch/templates/pgwatch-config.yaml index 1580de4..9fd7749 100644 --- a/helm/pgwatch/templates/pgwatch-config.yaml +++ b/helm/pgwatch/templates/pgwatch-config.yaml @@ -12,7 +12,7 @@ data: CMD="/pgwatch/pgwatch --sources=postgresql://$PGWATCH_USER:$PGWATCH_USER_PASSWORD@$METRIC_DATABASE_ENDPOINT:$METRIC_DATABASE_PORT/$METRIC_DATABASE_DATABASE?sslmode=$METRIC_DATABASE_SSLMODE" if [[ "$PG_IS_SINK" == "true" ]]; then - CMD="$CMD --sink=postgresql://$PGWATCH_USER:$PGWATCH_USER_PASSWORD@$METRIC_DATABASE_ENDPOINT:$METRIC_DATABASE_PORT/$METRIC_DATABASE_DATABASE?sslmode=$METRIC_DATABASE_SSLMODE" + CMD="$CMD --sink=postgresql://$PGWATCH_USER:$PGWATCH_USER_PASSWORD@$METRIC_DATABASE_ENDPOINT:$METRIC_DATABASE_PORT/$METRIC_DATABASE_DATABASE?sslmode=$METRIC_DATABASE_SSLMODE --retention=$PG_RETENTION_DAYS" fi if [[ "$PROM_IS_SINK" == "true" ]]; then diff --git a/helm/pgwatch/templates/pgwatch-deployment.yaml b/helm/pgwatch/templates/pgwatch-deployment.yaml index 27077aa..ed35c8a 100644 --- a/helm/pgwatch/templates/pgwatch-deployment.yaml +++ b/helm/pgwatch/templates/pgwatch-deployment.yaml @@ -56,6 +56,8 @@ spec: {{- if eq .Values.pgwatch.postgres.enable_pg_sink "true" }} - name: PG_IS_SINK value: "true" + - name: PG_RETENTION_DAYS + value: {{ .Values.pgwatch.postgres.settings.retention_days }} {{ end }} {{- if eq .Values.pgwatch.prometheus.enable_prom_sink "true" }} - name: PROM_IS_SINK diff --git a/helm/pgwatch/values.yaml b/helm/pgwatch/values.yaml index d3f2bc4..1a48648 100644 --- a/helm/pgwatch/values.yaml +++ b/helm/pgwatch/values.yaml @@ -3,6 +3,8 @@ pgwatch: namespace: pgwatch postgres: enable_pg_sink: "true" + settings: + retention_days: 31 # define is a database for the metrics needs to be created or if there is already an existing database create_metric_database: "true" #"false" new_metric_database: @@ -22,6 +24,7 @@ pgwatch: new_prometheus: create_prometheus: "true" image: "prom/prometheus:main" + data_retention: 30d volume: size: '10Gi' storageClass: 'crc-csi-hostpath-provisioner' From 94cfec15fc61ac0dfaca9c857bf8e777c248ba62 Mon Sep 17 00:00:00 2001 From: matthias Date: Mon, 7 Apr 2025 08:53:26 +0200 Subject: [PATCH 08/32] cleanup and change variable-name --- helm/pgwatch/templates/postgres-statefulset.yaml | 6 +++--- helm/pgwatch/values.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/pgwatch/templates/postgres-statefulset.yaml b/helm/pgwatch/templates/postgres-statefulset.yaml index 2a26a84..5d9c7ea 100644 --- a/helm/pgwatch/templates/postgres-statefulset.yaml +++ b/helm/pgwatch/templates/postgres-statefulset.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: postgres - image: {{ .Values.pgwatch.postgres.new_metric_database.image }} + image: {{ .Values.pgwatch.postgres.new_pg_database.image }} ports: - containerPort: 5432 name: postgres @@ -74,8 +74,8 @@ spec: pgwatch.pods.role: postgres spec: accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.pgwatch.postgres.new_metric_database.volume.storageClass }} + storageClassName: {{ .Values.pgwatch.postgres.new_pg_database.volume.storageClass }} resources: requests: - storage: {{ .Values.pgwatch.postgres.new_metric_database.volume.size }} + storage: {{ .Values.pgwatch.postgres.new_pg_database.volume.size }} {{ end }} \ No newline at end of file diff --git a/helm/pgwatch/values.yaml b/helm/pgwatch/values.yaml index 1a48648..a652f20 100644 --- a/helm/pgwatch/values.yaml +++ b/helm/pgwatch/values.yaml @@ -7,12 +7,12 @@ pgwatch: retention_days: 31 # define is a database for the metrics needs to be created or if there is already an existing database create_metric_database: "true" #"false" - new_metric_database: + new_pg_database: # Will be used for pgqwatch config only, if enable_pg_sink = "false" image: "docker.io/schmaetz/postgres:bookworm-17-1" volume: size: '10Gi' storageClass: 'crc-csi-hostpath-provisioner' - # use_existing_database: + # use_existing_database: # Will be used for pgqwatch config only, if enable_pg_sink = "false" # endpoint: postgresql.local # port: '5432' # database: PGWATCH_DATABASE From 00734cf10fee9d8bfa5a6365620e13f6b30ba06a Mon Sep 17 00:00:00 2001 From: matthias Date: Mon, 7 Apr 2025 14:22:20 +0200 Subject: [PATCH 09/32] added prom retention for new prometheus --- helm/pgwatch/templates/pgwatch-deployment.yaml | 2 +- helm/pgwatch/templates/prometheus-deployment.yaml | 10 +++++----- helm/pgwatch/values.yaml | 3 ++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/helm/pgwatch/templates/pgwatch-deployment.yaml b/helm/pgwatch/templates/pgwatch-deployment.yaml index ed35c8a..e7beddb 100644 --- a/helm/pgwatch/templates/pgwatch-deployment.yaml +++ b/helm/pgwatch/templates/pgwatch-deployment.yaml @@ -57,7 +57,7 @@ spec: - name: PG_IS_SINK value: "true" - name: PG_RETENTION_DAYS - value: {{ .Values.pgwatch.postgres.settings.retention_days }} + value: "{{ .Values.pgwatch.postgres.settings.retention_days }}" {{ end }} {{- if eq .Values.pgwatch.prometheus.enable_prom_sink "true" }} - name: PROM_IS_SINK diff --git a/helm/pgwatch/templates/prometheus-deployment.yaml b/helm/pgwatch/templates/prometheus-deployment.yaml index 4bb45dd..174e797 100644 --- a/helm/pgwatch/templates/prometheus-deployment.yaml +++ b/helm/pgwatch/templates/prometheus-deployment.yaml @@ -47,6 +47,10 @@ spec: timeoutSeconds: 1 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File + args: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time={{ .Values.pgwatch.prometheus.new_prometheus.settings.retention_days }}d" volumeMounts: - mountPath: /etc/prometheus name: prometheus-cm @@ -69,9 +73,5 @@ spec: replicas: 1 revisionHistoryLimit: 10 strategy: - rollingUpdate: - maxSurge: 25% - maxUnavailable: 25% - type: RollingUpdate - + type: Recreate {{ end }} \ No newline at end of file diff --git a/helm/pgwatch/values.yaml b/helm/pgwatch/values.yaml index a652f20..c55030c 100644 --- a/helm/pgwatch/values.yaml +++ b/helm/pgwatch/values.yaml @@ -24,7 +24,8 @@ pgwatch: new_prometheus: create_prometheus: "true" image: "prom/prometheus:main" - data_retention: 30d + settings: + retention_days: 31 volume: size: '10Gi' storageClass: 'crc-csi-hostpath-provisioner' From ce14f56fb2d240cb0ecabae6969122db0ed4a2a4 Mon Sep 17 00:00:00 2001 From: matthias Date: Mon, 7 Apr 2025 14:27:36 +0200 Subject: [PATCH 10/32] change default values --- helm/pgwatch/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/pgwatch/values.yaml b/helm/pgwatch/values.yaml index c55030c..9ad2663 100644 --- a/helm/pgwatch/values.yaml +++ b/helm/pgwatch/values.yaml @@ -22,7 +22,7 @@ pgwatch: prometheus: enable_prom_sink: "true" new_prometheus: - create_prometheus: "true" + create_prometheus: "false" image: "prom/prometheus:main" settings: retention_days: 31 From 5ec3259e7256043bd6009d384099863ad3847aea Mon Sep 17 00:00:00 2001 From: Mat thias <10142839+Schmaetz@users.noreply.github.com> Date: Mon, 14 Apr 2025 08:55:21 +0200 Subject: [PATCH 11/32] Update README.md --- helm/README.md | 164 ++++++++----------------------------------------- 1 file changed, 24 insertions(+), 140 deletions(-) diff --git a/helm/README.md b/helm/README.md index 1f2afbb..36413d2 100644 --- a/helm/README.md +++ b/helm/README.md @@ -4,156 +4,40 @@ [![Coverage Status](https://coveralls.io/repos/github/cybertec-postgresql/pgwatch/badge.svg?branch=master&service=github)](https://coveralls.io/github/cybertec-postgresql/pgwatch?branch=master) -# pgwatch v3-beta. Please test it as much as possible! - -This is the next generation of [pgwatch2](https://github.com/cybertec-postgresql/pgwatch2/). +# pgWatch-Helm-Charts +These Helm charts are available for a quick start on Kubernetes and Openshift. +Since the new pgWatch release (3.x), the Helm charts have been developed and tested primarily for Openshift. +The old Helm charts for pgwatch2 are deprecated and no longer maintained. Furthermore, they will be removed in summer 2025. + ## Quick Start +To use the Helm-Charts, you can either patch the repo onto your local system or install and update it directly using the Helm repository. +In either case, please familiarise yourself with the relevant values files before use and create a custom variant to set up pgWatch according to your preferences in your environment. -To fetch and run the latest **demo** Docker image, exposing -- Grafana on port 3000, -- the administrative web UI on port 8080, -- the internal configuration and metrics database on port 5432: - -```shell -docker run -d --name pw3 -p 5432:5432 -p 3000:3000 -p 8080:8080 -e PW_TESTDB=true cybertecpostgresql/pgwatch-demo -``` - -After some minutes you could open the ["Database Overview"](http://127.0.0.1:3000/dashboard/db/db-overview) dashboard and start looking at metrics. For defining your own dashboards you need to log in Grafana as admin (`admin`/`pgwatchadmin`). - -If you don't want to add the test database for monitoring, remove the `PW_TESTDB` parameter when launching the container. - - - -## Development and production use - -For production and long term installation `cybertecpostgresql/pgwatch` Docker image should be used. -For the fastest development and deployment experience the Docker compose files are provided: - -```shell -git clone https://github.com/cybertec-postgresql/pgwatch.git && cd pgwatch - -docker compose -f ./docker/docker-compose.yml up --detach -``` -
- ✔ Network pgwatch_default       Created
- ✔ Container pgwatch-postgres-1  Healthy
- ✔ Container pgwatch-pgwatch-1  Started
- ✔ Container pgwatch-grafana-1   Started
-
- -These commands will build and start services listed in the compose file: -- configuration and metric database; -- pgwatch monitoring agent with WebUI; -- Grafana with dashboards. +### Helm-Repository +comming soon -## Monitor Database +### git clone +```sh +git clone https://github.com/cybertec-postgresql/pgwatch-charts.git +cd pgwatch-chart/helm/pgwatch -After start, you could open the [monitoring dashboard](http://localhost:3000/) and start -looking at metrics. -To add a test database under monitoring, you can use [built-in WebUI](http://localhost:8080/). Or simply -execute from command line: -```shell -docker compose -f ./docker/docker-compose.yml up add-test-db --force-recreate -``` -
-[+] Running 2/0
- ✔ Container pgwatch-postgres-1     Running                                                                       0.0s
- ✔ Container pgwatch-add-test-db-1  Created                                                                       0.0s
-Attaching to pgwatch-add-test-db-1
-pgwatch-add-test-db-1  | BEGIN
-...
-pgwatch-add-test-db-1  | GRANT
-pgwatch-add-test-db-1  | COMMENT
-pgwatch-add-test-db-1  | INSERT 0 1
-pgwatch-add-test-db-1 exited with code 0
-
- -## Produce Workload +// Install helm-Chart +helm install pgwatch -n pgwatch -f custom-values.yaml . -To emulate workload for added test database execute: -```shell -docker compose -f ./docker/docker-compose.yml up pgbench ``` -
-[+] Running 2/2
- ✔ Container pgwatch-postgres-1  Running                                                                          0.0s
- ✔ Container pgwatch-pgbench-1   Created                                                                          0.1s
-Attaching to pgwatch-pgbench-1
-pgwatch-pgbench-1  | dropping old tables...
-pgwatch-pgbench-1  | NOTICE:  table "pgbench_accounts" does not exist, skipping
-pgwatch-pgbench-1  | NOTICE:  table "pgbench_branches" does not exist, skipping
-pgwatch-pgbench-1  | NOTICE:  table "pgbench_history" does not exist, skipping
-pgwatch-pgbench-1  | NOTICE:  table "pgbench_tellers" does not exist, skipping
-pgwatch-pgbench-1  | creating tables...
-pgwatch-pgbench-1  | generating data (client-side)...
-pgwatch-pgbench-1  | 100000 of 5000000 tuples (2%) done (elapsed 0.11 s, remaining 5.17 s)
-pgwatch-pgbench-1  | 200000 of 5000000 tuples (4%) done (elapsed 0.25 s, remaining 6.06 s)
-...
-pgwatch-pgbench-1  | 5000000 of 5000000 tuples (100%) done (elapsed 16.28 s, remaining 0.00 s)
-pgwatch-pgbench-1  | vacuuming...
-pgwatch-pgbench-1  | creating primary keys...
-pgwatch-pgbench-1  | done in 42.29 s (drop tables 0.03 s, create tables 0.04 s, client-side generate 18.23 s, vacuum 1.29 s, primary keys 22.70 s).
-pgwatch-pgbench-1  | pgbench (15.4)
-pgwatch-pgbench-1  | starting vacuum...
-pgwatch-pgbench-1  | end.
-pgwatch-pgbench-1  | progress: 5.0 s, 642.2 tps, lat 15.407 ms stddev 11.794, 0 failed
-pgwatch-pgbench-1  | progress: 10.0 s, 509.6 tps, lat 19.541 ms stddev 9.493, 0 failed
-...
-pgwatch-pgbench-1  | progress: 185.0 s, 325.3 tps, lat 16.825 ms stddev 8.330, 0 failed
-pgwatch-pgbench-1  |
-pgwatch-pgbench-1  |
-pgwatch-pgbench-1  | transaction type: builtin: TPC-B (sort of)
-pgwatch-pgbench-1  | scaling factor: 50
-pgwatch-pgbench-1  | query mode: simple
-pgwatch-pgbench-1  | number of clients: 10
-pgwatch-pgbench-1  | number of threads: 2
-pgwatch-pgbench-1  | maximum number of tries: 1
-pgwatch-pgbench-1  | number of transactions per client: 10000
-pgwatch-pgbench-1  | number of transactions actually processed: 100000/100000
-pgwatch-pgbench-1  | number of failed transactions: 0 (0.000%)
-pgwatch-pgbench-1  | latency average = 18.152 ms
-pgwatch-pgbench-1  | latency stddev = 13.732 ms
-pgwatch-pgbench-1  | initial connection time = 25.085 ms
-pgwatch-pgbench-1  | tps = 534.261013 (without initial connection time)
-pgwatch-pgbench-1  | dropping old tables...
-pgwatch-pgbench-1  | done in 0.45 s (drop tables 0.45 s).
-pgwatch-pgbench-1 exited with code 0
-
- -## Inspect database - -> [!IMPORTANT] -pgAdmin uses port 80. If you want it to use another port, change it in `docker-compose.yml` file. -To look what is inside `pgwatch` database, you can spin up pgAdmin4: -```shell -docker compose -f ./docker/docker-compose.yml up --detach pgadmin +### Custom Values-file +```sh +// Edit Values File +cp values.yaml custom-values.yaml +vi custom-values.yaml ``` -Go to `localhost` in your favorite browser and login as `admin@local.com`, password `admin`. -Server `pgwatch` should be already added in `Servers` group. - -## Development - -If you apply any changes to the source code and want to restart the agent, it's usually enough to run: - -```shell -docker compose -f ./docker/docker-compose.yml up pgwatch --build --force-recreate --detach -``` - -The command above will rebuild the `pgwatch` agent from sources and relaunch the container. - -## Logs - -If you are running containers in detached mode, you still can follow the logs: -```shell -docker compose -f ./docker/docker-compose.yml logs --follow -``` - -Or you may check the log of a particular service: -```shell -docker compose -f ./docker/docker-compose.yml logs pgwatch --follow +### Check the Pods +```sh +kubectl get pods -n pgwatch +oc get pods -n pgwatch ``` # Contributing From 820b22446622e7de1cad972d4d916c6c801602b9 Mon Sep 17 00:00:00 2001 From: Mat thias <10142839+Schmaetz@users.noreply.github.com> Date: Mon, 14 Apr 2025 09:01:40 +0200 Subject: [PATCH 12/32] Create README.md --- helm/pgwatch/README.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 helm/pgwatch/README.md diff --git a/helm/pgwatch/README.md b/helm/pgwatch/README.md new file mode 100644 index 0000000..8a4327f --- /dev/null +++ b/helm/pgwatch/README.md @@ -0,0 +1,38 @@ +[![Documentation](https://img.shields.io/badge/Documentation-pgwat.ch-brightgreen)](https://pgwat.ch) +[![License: MIT](https://img.shields.io/badge/License-BSD_3-green.svg)](https://opensource.org/license/bsd-3-clause) +[![Go Build & Test](https://github.com/cybertec-postgresql/pgwatch/actions/workflows/build.yml/badge.svg)](https://github.com/cybertec-postgresql/pgwatch/actions/workflows/build.yml) +[![Coverage Status](https://coveralls.io/repos/github/cybertec-postgresql/pgwatch/badge.svg?branch=master&service=github)](https://coveralls.io/github/cybertec-postgresql/pgwatch?branch=master) + + +# pgWatch-Helm-Chart +This Helm chart allows you to set up the pgWatch stack using helm in containers or distributions such as Openshift. +Note: This Helm chart is developed and tested primarily for Openshift. + +## Quick Start +To use the Helm-Charts, you can either patch the repo onto your local system or install and update it directly using the Helm repository. +In either case, please familiarise yourself with the relevant values files before use and create a custom variant to set up pgWatch according to your preferences in your environment. + +### Helm-Repository +comming soon + +### git clone +```sh +git clone https://github.com/cybertec-postgresql/pgwatch-charts.git +cd pgwatch-chart/helm/pgwatch + + +// Install helm-Chart +helm install pgwatch -n pgwatch -f custom-values.yaml . + +``` + +## customisation +The Helm chart currently supports PostgreSQL and Prometheus as a sink. This can be controlled via the [values](https://github.com/cybertec-postgresql/pgwatch-charts/blob/pgwatch-3-helm-chart/helm/pgwatch/values.yaml) file. +- PostgreSQL + - Use an existing configuration and metric database + - Create a new PostgreSQL-Instance in the same namespace +- Prometheus + - Use an existing Prometheus as Sink (enables Sink-Connect on Port 9188) + - Create a new Prometheus-Instance in the same namespace + + From d92fcfcfa5b789ca30c43b157f9d74f5a3c5c40f Mon Sep 17 00:00:00 2001 From: Mat thias <10142839+Schmaetz@users.noreply.github.com> Date: Mon, 14 Apr 2025 09:05:13 +0200 Subject: [PATCH 13/32] Update README.md --- helm/pgwatch/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helm/pgwatch/README.md b/helm/pgwatch/README.md index 8a4327f..3195f3a 100644 --- a/helm/pgwatch/README.md +++ b/helm/pgwatch/README.md @@ -34,5 +34,9 @@ The Helm chart currently supports PostgreSQL and Prometheus as a sink. This can - Prometheus - Use an existing Prometheus as Sink (enables Sink-Connect on Port 9188) - Create a new Prometheus-Instance in the same namespace +- Grafana + - Deploy Grafana with the dashboards for PostgreSQL as a sink +## Limitation +Please note that the Grafana dashboard was developed for use with PostgreSQL as a sink. If you decide to use Prometheus as a sink, you will need to build your own dashboards and configure Prometheus as a data source in Grafana yourself. If you want to use your own systems behind Prometheus, you can use Yaml to easily prevent Grafana from being deployed. From b472a0424aab8a5f56bcaade2028222a0e0ed754 Mon Sep 17 00:00:00 2001 From: Mat thias <10142839+Schmaetz@users.noreply.github.com> Date: Mon, 14 Apr 2025 09:06:17 +0200 Subject: [PATCH 14/32] Update values.yaml --- helm/pgwatch/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helm/pgwatch/values.yaml b/helm/pgwatch/values.yaml index 9ad2663..682fd6d 100644 --- a/helm/pgwatch/values.yaml +++ b/helm/pgwatch/values.yaml @@ -29,3 +29,5 @@ pgwatch: volume: size: '10Gi' storageClass: 'crc-csi-hostpath-provisioner' + grafana: + enable_grafana: "true" From e54264a5d81c0e751dcbf71265f4c4e269312a5e Mon Sep 17 00:00:00 2001 From: Mat thias <10142839+Schmaetz@users.noreply.github.com> Date: Mon, 14 Apr 2025 09:07:09 +0200 Subject: [PATCH 15/32] Update prometheus-config.yaml --- helm/pgwatch/templates/prometheus-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helm/pgwatch/templates/prometheus-config.yaml b/helm/pgwatch/templates/prometheus-config.yaml index d023553..f6ec135 100644 --- a/helm/pgwatch/templates/prometheus-config.yaml +++ b/helm/pgwatch/templates/prometheus-config.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.pgwatch.prometheus.new_prometheus.create_prometheus "true" }} apiVersion: v1 kind: ConfigMap metadata: @@ -17,3 +18,4 @@ data: - job_name: 'pgwatch' static_configs: - targets: ['pgwatch-svc.pgwatch.svc.cluster.local:9188'] +{{ end }} From 8da830922056c6e8c450e6d51d234da8f9b88c2c Mon Sep 17 00:00:00 2001 From: Mat thias <10142839+Schmaetz@users.noreply.github.com> Date: Mon, 14 Apr 2025 09:08:26 +0200 Subject: [PATCH 16/32] Update grafana-deployment.yaml --- helm/pgwatch/templates/grafana-deployment.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/helm/pgwatch/templates/grafana-deployment.yaml b/helm/pgwatch/templates/grafana-deployment.yaml index aa70ecb..5f92407 100644 --- a/helm/pgwatch/templates/grafana-deployment.yaml +++ b/helm/pgwatch/templates/grafana-deployment.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.pgwatch.grafana.enable_grafana "true" }} apiVersion: apps/v1 kind: Deployment metadata: @@ -131,4 +132,5 @@ data: disableDeletion: false updateIntervalSeconds: 10 #how often Grafana will scan for changed dashboards options: - path: /var/lib/grafana/dashboards \ No newline at end of file + path: /var/lib/grafana/dashboards +{{ end }} From cbcf502e1bf824ba0dccc6147f115354664f9ac8 Mon Sep 17 00:00:00 2001 From: Mat thias <10142839+Schmaetz@users.noreply.github.com> Date: Mon, 14 Apr 2025 09:20:50 +0200 Subject: [PATCH 17/32] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 56f9ad9..1067783 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# pgwatch2-charts +# pgWatch-charts -Chart templates for cybertec-postgresql/pgwatch2 +Chart templates for cybertec-postgresql/pgwatch ## Disclaimer From cf52c36ac5f63210c6e1ddc56201a8780e8e4fe7 Mon Sep 17 00:00:00 2001 From: Mat thias <10142839+Schmaetz@users.noreply.github.com> Date: Mon, 14 Apr 2025 09:33:37 +0200 Subject: [PATCH 18/32] Update values.yaml --- helm/pgwatch/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/pgwatch/values.yaml b/helm/pgwatch/values.yaml index 682fd6d..440622e 100644 --- a/helm/pgwatch/values.yaml +++ b/helm/pgwatch/values.yaml @@ -23,6 +23,7 @@ pgwatch: enable_prom_sink: "true" new_prometheus: create_prometheus: "false" + create_alertmanager: "false" image: "prom/prometheus:main" settings: retention_days: 31 From d9ee500c38d5c8de20a3241add02d69a9619ddae Mon Sep 17 00:00:00 2001 From: Mat thias <10142839+Schmaetz@users.noreply.github.com> Date: Mon, 14 Apr 2025 09:42:56 +0200 Subject: [PATCH 19/32] Create alertmanager_deployment not ready --- .../pgwatch/templates/alertmanager_deployment | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 helm/pgwatch/templates/alertmanager_deployment diff --git a/helm/pgwatch/templates/alertmanager_deployment b/helm/pgwatch/templates/alertmanager_deployment new file mode 100644 index 0000000..4cf83d1 --- /dev/null +++ b/helm/pgwatch/templates/alertmanager_deployment @@ -0,0 +1,77 @@ +{{- if eq .Values.pgwatch.prometheus.new_prometheus.create_alertmanager "true" }} +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: pgwatch + vendor: opensource.cybertec + postgres-operator.cybertec.at/stack: pgwatch + name: pgwatch-alertmanager +spec: + selector: + matchLabels: + postgres-operator.cybertec.at/stack: pgwatch + name: pgwatch-alertmanager + template: + metadata: + creationTimestamp: null + labels: + postgres-operator.cybertec.at/stack: pgwatch + name: pgwatch-alertmanager + spec: + containers: + - image: {{ .Values.pgwatch.prometheus.new_prometheus.image }} + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /-/healthy + port: 9090 + scheme: HTTP + initialDelaySeconds: 15 + periodSeconds: 20 + successThreshold: 1 + timeoutSeconds: 1 + name: pgwatch-alertmanager + ports: + - containerPort: 9090 + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /-/ready + port: 9090 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + args: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time={{ .Values.pgwatch.prometheus.new_prometheus.settings.retention_days }}d" + volumeMounts: + - mountPath: /etc/prometheus + name: prometheus-cm + - mountPath: /prometheus + name: prometheus-volume + securityContext: {} + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + terminationGracePeriodSeconds: 30 + volumes: + - configMap: + defaultMode: 420 + name: pgwatch-prometheus-cm + name: prometheus-cm + - name: prometheus-volume + persistentVolumeClaim: + claimName: prometheus-pvc + progressDeadlineSeconds: 600 + replicas: 1 + revisionHistoryLimit: 10 + strategy: + type: Recreate +{{ end }} From 958ca285a051bf4588a1d244c5fb589d071c3c40 Mon Sep 17 00:00:00 2001 From: Mat thias <10142839+Schmaetz@users.noreply.github.com> Date: Tue, 15 Apr 2025 07:49:13 +0200 Subject: [PATCH 20/32] Create charts-folder --- charts | 1 + 1 file changed, 1 insertion(+) create mode 100644 charts diff --git a/charts b/charts new file mode 100644 index 0000000..b498fd4 --- /dev/null +++ b/charts @@ -0,0 +1 @@ +/ From 8ff62e9f1fdc73f2b181143ec608c54fc393fe3b Mon Sep 17 00:00:00 2001 From: Mat thias <10142839+Schmaetz@users.noreply.github.com> Date: Tue, 15 Apr 2025 07:49:34 +0200 Subject: [PATCH 21/32] Delete charts --- charts | 1 - 1 file changed, 1 deletion(-) delete mode 100644 charts diff --git a/charts b/charts deleted file mode 100644 index b498fd4..0000000 --- a/charts +++ /dev/null @@ -1 +0,0 @@ -/ From a238476d7dd615c12242a3994f7c510d8d4adfa5 Mon Sep 17 00:00:00 2001 From: Mat thias <10142839+Schmaetz@users.noreply.github.com> Date: Tue, 15 Apr 2025 08:04:14 +0200 Subject: [PATCH 22/32] Create index --- charts/index | 1 + 1 file changed, 1 insertion(+) create mode 100644 charts/index diff --git a/charts/index b/charts/index new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/charts/index @@ -0,0 +1 @@ + From f90c9995b0fca043a8255849f038c3ae7cc4b55c Mon Sep 17 00:00:00 2001 From: Mat thias <10142839+Schmaetz@users.noreply.github.com> Date: Tue, 15 Apr 2025 09:59:36 +0200 Subject: [PATCH 23/32] Update values.yaml --- helm/pgwatch/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helm/pgwatch/values.yaml b/helm/pgwatch/values.yaml index 440622e..6f2f837 100644 --- a/helm/pgwatch/values.yaml +++ b/helm/pgwatch/values.yaml @@ -32,3 +32,6 @@ pgwatch: storageClass: 'crc-csi-hostpath-provisioner' grafana: enable_grafana: "true" + enable_datasources: + postgres: "true" + prometheus: "false" From 188e41a28c1ab7832e4b3f7685e9fb1d71ed3b82 Mon Sep 17 00:00:00 2001 From: Mat thias <10142839+Schmaetz@users.noreply.github.com> Date: Tue, 15 Apr 2025 10:00:50 +0200 Subject: [PATCH 24/32] Update grafana-deployment.yaml --- helm/pgwatch/templates/grafana-deployment.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/helm/pgwatch/templates/grafana-deployment.yaml b/helm/pgwatch/templates/grafana-deployment.yaml index 5f92407..b187a06 100644 --- a/helm/pgwatch/templates/grafana-deployment.yaml +++ b/helm/pgwatch/templates/grafana-deployment.yaml @@ -29,11 +29,11 @@ spec: value: Admin - name: GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH value: /var/lib/grafana/dashboards/1-global-db-overview.json - - name: GF_DATABASE_TYPE - value: postgres - - name: GF_INSTALL_PLUGINS - value: marcusolsson-treemap-panel + # - name: GF_INSTALL_PLUGINS + # value: marcusolsson-treemap-panel {{- if eq .Values.pgwatch.postgres.create_metric_database "true" }} + - name: GF_DATABASE_TYPE + value: postgres - name: GF_DATABASE_HOST value: postgres-svc:5432 - name: GF_DATABASE_NAME @@ -51,6 +51,8 @@ spec: name: pgwatch-postgresql-secret-pgwatch key: password {{- else }} + - name: GF_DATABASE_TYPE + value: postgres - name: GF_DATABASE_HOST value: {{ .Values.pgwatch.postgres.use_existing_database.endpoint }} - name: GF_DATABASE_NAME From deffb81a1fb52ae17d56bb371795b2ccac357d75 Mon Sep 17 00:00:00 2001 From: matthias Date: Tue, 15 Apr 2025 16:21:59 +0200 Subject: [PATCH 25/32] added chart repo & updates version --- charts/index.yaml | 13 +++++++++++++ charts/pgwatch-3.0.1.tgz | Bin 0 -> 94955 bytes helm/pgwatch/Chart.yaml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 charts/index.yaml create mode 100644 charts/pgwatch-3.0.1.tgz diff --git a/charts/index.yaml b/charts/index.yaml new file mode 100644 index 0000000..a11d4cc --- /dev/null +++ b/charts/index.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +entries: + pgwatch: + - apiVersion: v1 + appVersion: "3.2" + created: "2025-04-15T16:21:29.294425244+02:00" + description: A Helm chart for pgwatch monitoring tools + digest: 69410525096ac64bac0f782205d92485e0d57cfc72f2e1d1684933baf3c21100 + name: pgwatch + urls: + - https://cybertec-postgresql.github.io/pgwatch-charts/pgwatch-3.0.1.tgz + version: 3.0.1 +generated: "2025-04-15T16:21:29.288927563+02:00" diff --git a/charts/pgwatch-3.0.1.tgz b/charts/pgwatch-3.0.1.tgz new file mode 100644 index 0000000000000000000000000000000000000000..fec39ce253f43e888c98d78c6f29519b5b620883 GIT binary patch literal 94955 zcmZsiV{j(TvjiT8eY1T0u71>UtIpt*7HHL}O!eI_GO4WV7o$ zM1mP03yKe8)whU;2-At)koHjh_(itm3!xP|5`#RnRhvj~aQ*gCX{?q6u|G#OA4i=& zZMONdn2OGcm zAEE-E^MynEyf=$CQTDdqS)MxASfwuw3pS}gn<;MzX^pqkapz#k(p)MnvPeFaxMm1 zD8YK5aghLl{rUcVV(|AULqbsXC{Rd^GxZe6Q8uu@fqYQZgJA}^Z^&7ovFO9)>&V2f z_{gB%hd*^aQ*tS&ts<*w2{IxTbbGI1nRPm3!p4J_7biGCDvqwj3>*Jcd zne0QiP#=5xE0r%W+~J0@RCWcc6-?x&scz{kX1kg{9y z4Io0}?Z(O|KNG)*u=g2})T6-wY;}of_Q7Z=jezwZc%je5Z6RDB?ih>++T$FZmRMph zrqU3iH?YoUlw4=xo_5bv0gT`HwEzLPlY4DFn&wCC-1F-uQfR z6VIOLM^J0vO@%Qb%8-G@{K+-FM}F8`J;Ca$y|25+^Xp+`;zs8*W_ME_eQGJD_bRXl zxB?@7>@By3`=aY|5jmDvDiKPS?mJX9eoGYYuMJ>ALSY%q2aslS@g%wuk|S}FL?>2v zSxfz)>E4$qGYEqU%S-T==tbwI{I+8x{Z7$juexoGf>xM1^5Z=DFr}gLHSyvT-YX#P z_+D!FwvUH=g=VZDdE6sMbzhvtP%%O@RaOiH%>p4 za`SdvKuCgE#+v^xG=Con#vA$Vy>mdNOr7^vl) z`-0^mjN5^>82ngw)v~+4qOjX~dNcwZIEi3z*~vB_G4eG4B&E4vf{gP|(ozEM~`)0)>j51fy zf@Z0G2tvsd|K2R;6heik2@TtQ29s{EvDKD_ThE2`=kI}u-$|98vJ=a)-k?K7&m&_)X03;DoUFFm zo@{WLEAx8LeQnmgY6!_?S?VwfU3-VT?rjK%_LkF6mUeWmU28%zsS>IiK4ARbO%6kb zsrJPAiS}0xujwUIQ?#_o?3k(>^Gfy?j2X3V+L+Sutwr+(E@FLtUE!?({fdc3$yo4O zjo1s|ftOiv4e(`-lL5tyi0h0U^3waLe`2%&zTZXH-2I7CSsSM}RSw2pku{#Wcmi6v zN#8qv=qu8GjjZiO6?hx>e zwL`S3!e=~vh{!WE=m58>-o5ZPLXn|sjPi|DLOGyN{M?@2y7`@WfTYsd4akfiws!+2caZ;%DxT_|B)1=sO2Ged3*S55=w$s77i{wxCH%oE;P9@=(~$V(A^TUcNnj^N9~6ItXF-uaYD{x{j*zI)Qu6^Ak+D{f?<}BdL#g}qyh0r%{K>{T)=A$!Oh!% z0fCJx&J61$l(U-~uDsW;+`L%_#6*NrYq{{eZz)EAT74f#7NUJ~z-7td>SitYz1<0;MRz{<9#P!93DW zoC1C4{Z~3CFe3sk4S%KZ6Ry3`*qhI1vj<86h|VUKsUZb68sq3Im8yNom7#5Kh3rCi=VTYqdD~uA40Sr2~cCHQ}vm z7XHr_YalsRrS}I5st)F#8WpM*h-#~}2=~|p@UFc5MJZA>xW74Q;D_LA9>)5Rcn7)v z{^0B12LLH>cIGtEb9tRpAL>_*C8{3*2~gvL&U>*cE?aPfDlbMee6)+=XdC*B30=iQ ztV%V0N6UwN)h+tQXToaZS!y;~FRrELL%g4=S|pex-rxOth78IusIDCuCvss1wrL%o zIg&>5Iua5XVZea!wT7UEiy#qus z>YMu^s_XF^<@oyf8^zGKdamjxKF0E+fmn((0~zT2%WzV>cH8DwjlYJwO0Pdrj?BK9 z0pM}+u0jg~Pq=V7YP2?ZS|_2>4o;!cYB%ls&3kG_r$Onp+A(j1B36dlUE0fh8g<4> zDQqp5D&x{d04VV%c?@WXjG6hp@uxrVC)>Dst6VMU3^^Sy6#!i7s*u4=H-&i(bGLbg zuTrAQ$pP+-2=Qvw?ie4!WcFy=$v63Bl~-)aO7AMNw|NmNe7mT>RyVJk^7L}gdi-~u zJV3XY3H!D&zxAD5B>rsw^}`12jbFn^*R{Ayd$&Vl$B=+`!ibQ>`H5XC$A|4*AT>j6 zc!8k%k0cRcLNOs3;?(X;n}ddmcbdOmNSuBSrXO(w2k?UTQMzSGN1PxAq&_XJf7I4d;?6}_cO6465l$!O%Y%F4y! zuYRH($B}MHWUeTrUGd<^Vf3z3_Rzx0^~||9WV8<_;>cU_lwa?4xxBbuk)$%>f34Kr z6k(-_!$~0qGii!OQYuFc3?^MLYS_wDZqM)=lZ~@h!L1^}Zh%K}?U1ma#@EyKF~VW8 zZu;|Q?*6twtt;j>0Q^O?J{s=XkCU_h;vLEg$W%Xb;(S{);tZ*mu|(GX7Y z%nDNGKhzNx1nK~%V*&c`5wH{&yaaxrJ2-DjT<%(6#06{H2L~mPvUQ$x-z$EZ!pOo5 z>1oYCd5;Jg*UJK)7Gc_~!whtl-srQcaVpPxM~Fpd;DkI*7+K2BSeEMi1xyUgnmg8I=J0vLvYaz z%6gH;(sB)5z$Nl#_+h|E29$!LbRq_jQ{oxE?zXkbGWO^BAvuU9$R0XN_6n@UdJMxb`Gaunv;Xv)o;8h&!Fb2xDG{SZrmf^21R7H04u|&EA%9E^S|9#1DPy#JQqo zUh>xi|44;@@>2oz%_OT-&N0k8_J5`-Iraz6fE$5 zpQy+u-FMA&A-bKSI!AYzneVmwckRC`65B<08QAXmU_s>j9lD1Zgem-n*h2ViHi}LS zyZR&F9%v3goDLn3<=nXmcxr*#8UL+^gA5&ONmEnrjuKh8W^hRjRPJUV5;8-yUv7NG z$+PmO8_|Fj$VHrniUeI>cEKT@9wm94}keyFxe7`R&TcdyyDuj94%LECj0 zCK0HKQ2(`B^+Dr1Ywibm)fKf1M}8LNpib}zl&tED{|#C+jnyQvcOUN(6i$oRH!v+7^ya(=lrj34V-I+C z%LUwCTK=y6{WfIa8~wz(Lr?UN%&*v*YyYQ!0N2V^P0{+)k|XEH7~+Qa=H~q&30SpE>4W}HckvZe2iUMVKe|Z za6op-uA!IJ@!8b9bZjk%%jRGRxGXS`f9px^yQe;LaGqcYk#&L~G<3I(o2tXS6Tuga zntHk6^dZ$ePU_L)Ev{v}*`!JeWNvlTpwczB87~?XLexm*fh>+I&boffxgQGm;phG} zGS>W5wP-c@b5H3&6d8@Kz}lgB!6kyzQ{4RSai`eYeRkOJiy z%>f0Q1QNkzlP)UFCR;z#tojZBh(QLA!1x3X^Uer)AifB*<}Lk}(Z;4vJO~nk6*ok* z*YSvKh!Fy@1~wJLpk5R7&8k10AL_e@4?uDG4f6L?tQNzSumA*Ku1@yAGLoG(!HPa1 z7%`AR0Gs#NGalk0e47K&^MHx6tJd^XuevI(}LVF z>mWrV(cw%GB1J=ipTj?3JSryCT^qdJVH~LsixJp`mp_<+-lcgS98TdX9$-i<2XT@E zez|zkU4YNbKN$*CN0w52k2{4QBkY#)6c0I?J^xufUV1B@{ae`0g%Gj)8)exmQ}q_} zER6~kV|nR!1c3X`1&A$BqY+c|U-^h_)3CCq+h$@$l!rQfw}|oQB2A7+Bofz-tQukA z(ZV$T3w`&b;plc3l+}#%KX$kjq|ZFjIh|uK0Gf~=O6nmrkBP)v_Ey`^59^t_Ve#uq zmzi**J^(KG=~SO;?Gn%i`|P@+FT*!ZSPBmf+A?jprHhbD@b3HM?BVH9TwgQXfz;Qb zhniPeSFZ7AA$TVwDDYGOI8!2Qe#Ub}UTn}WMtL485bVHq+lJdEr#jjkkBJZI$_#r} zO7S-FJhr`HQE!jiDT~`FukbW>yh2IuEw>v_al{$C6PFy{DO_gFUl67dd^QGBiN||2 zsGv_=oq))$N0!(O+5T*ch$%-2aPIFKM zM<0-rE&jYkt?ZYV=*#Pqr=WmA(%UisgF*XdMnb;@o=9(SnNHHGRAtS;J7%*KvZ{6wJPC5(UE=t z8S?v>Q)=0Q;^b?*3uMd>3b}I>1E(oP zwBL8rZU(wCBIcE|WQM9KaGfb1ypF_!KoKdU04bw@NhB5Yam+EKXhU4~Ax+&!D^?ff zxlx13_(FRZ4Oo;*6+>fo$PpTegzDVia^$8`CyTQ2ZQGshiJyfO<$&Rd2JdOB8$T)& z6|_K)Zy7ya!d`_pMrp+2uD>WpsJ^2~mX>j%#>YJ>u2_SQb;*$Gu4sbG=9!!j{Jqmj zPf!0iFr7O`jZoylKk?egx&)FLwMV2TJjnoqoYni0L_guUlZ_i98wOeS^%C)SQ9K&A zepsz9;&>6tss)<{-yl!I@KO>xldz?E`vv4~7r-OtCcMN?VzS&PzRUa&Rc__d2mKvp z;OdH=vC8zEE&g34V@DEfpBxloCRXBaZetZ}}vZ97H*N8BJ z)0j-ZJDOlz{|#y5(_bgTV0D7bKRg?&PgzwW-f}iJhlF4L=4fnfsN2)=PUs2B#+N9> zykT2%EOcPaK>A62bVF2u!{2t&(&omTub;5R>2z7a!|qe{IT%}3GG`HN;E1AVU$I&O zEoCG_zYNXG5#qSJ`LgrURqs4!yz%*Qpm8}JDRz$Y?>QiYTArrMcsIe5i11OxkOC%j zYWEwIjmyoXX0}X9+Yq&8>aF%{MRFjuu#54IEHgl5(ismchEeh9_p{iAP;~qnpm5Nc z#06n(@Wv*`8R54To#WxlK+dUF*hGtNuLN_H!qBP^Ub_S*i3)8zqRWQP0;0ifCi8sP zFVkc6C5z!vA1OqmC~Xe+%$Ag(6;k~~Nea0fSrih)a z>?u&(b|*G^?ibtmw2WvUucDyIDdz%n39nsm(}GZKDm5&RPqL~3kkn4yE*JC3*qQ1+ zv9`rH8l6vSr%%N`f$`@riwHK6HlQKwlRtN2Jeuyt#(5TbU3L7l!_#^Pe-v~~*$-y; zjA`n(o3X+1cyAMI0!Lj9@mg<<$(gC(wA**lp9ht4YtddG z`VXm#@3rDxRZnQ%U(YWpvlH3j9!=EH#8IXAHy9r~fWzPuqb4iCL~`8X+}TqAzEibb z7D_)9AG|b-$;LldG}QL@&nt+@roQj|{GqE&>m}LG#hGigIu8W~`&(3~YEow6w@ktz z>^G5j$j6nkR_9Gu37_oG%az|N_ugb3x}PNRg?@WFqF(~mb~t|f_E{{8Y(CP5h*Qf9 z*rmw4wxm_0BRM!ar1or9EJWQiYCU5#0e?jfG54xeDbn3l4pBE~VCvE8o<4@(XX8>D zN3-u~7Hnh3#xT|0Gw*r!O~=7Wov*owB1PxSQV%=Utn*x7FKS1Z>2yr{Ts0lg?-ht>H|*Ok92J}cNaM^t6)dXUz=;k+(+GS6#o?OGxq zAXt|+9s#}ua}=oWqLVGR)kEzK8?QxitQ)Vee%0^+J}$oBY@Y7l9LIVJ$du++)sz`4 z!5GzjaIMeXtCB_sPRVx*lik2wE!7NV#4jzlxGZu?3e|FAN}jA@H7hVbSZQ)f zEvGF@=`QrEgUerpe>j_7h!EQ`XUBrQmH45}+U3})ULhbqPXBOR$5;E6P|HZ%GY02R zcJ;}7#d)uB-Swvgg5C~ewgqef^BtSSLK+`|MufzfUYt7cp(#Du5oKPXCkq6@E1g!S z-HTnM;ilDar_JexQVZUTgB#>}S9zaH67k0t$ZxqfC8{O~8VCww#7!xPc82m~7c_`Z zwHaiOpg{n)d**#lM%>o5lVSnr`j?-2w6d{Cwc#I!p%1Ba|7qc(W9W%=3cr(x-!E(s zu`q9&Kc2kwHjRqQ;A`{EptP%c!5_Tw_L^b?ysRWBClgU~hx#r9QL#yQUeIGEvLNWs zLHPXB52af{$y*kJrW@}WB*0Buj|*`Noh&SGg{21nlpZ~cwo;9hw&`)X+0wX@N+XKg zaa|-ok8|r7{~ZlKk<3ql=UkIU#PGMVYF76@M2CrHarI21B}BA-#L<~5P*SPiL|2~O z(Y{4`U}N>0QB>MLZ2RT@8jdY>OTkq*#Q1DeaEPHyi^*C>_NxJHC4@RMLB>70(8~K} zik6!qZ%yu1_Anpb@w@1xS!*AhDz_SC(xD3^h8UOjXGih%Om>U2hpVNqR)@JIv9!%| zFsbH|!jbp@3i!c5H-X@%FM+Qh1MKzIc{8ISP2TJIgFBC{#-k5_$(j{Bx4|InSgqCM zSr<3)q1oXoau*>WbTKM?Sdh6~m>c#>LpQCTr`4{Otnhv89^LJn+_iZ;Jii|vyhhIR zXNa5W$z`VYcH7OaZ8w9Cn%LDw?~+(O<$k`{wc!|W#74Hg-NVZv3;T#L0$z3fM|o@Y zaq*ItUIw&X<@V%tC&zOsCM|TA=pNJfR?Ah4t%`$Fmo7OOZ3Be=yBT?ip9e!<+{Q14xTUO4Esl)p7(7 z>;H}p18wmbRp=oR=pECQ5@}StlOik(q{hbOF02o4?#}L>VCDFw6W)<7npdKD)0qaldQ_J3tZpa;DZAwc&&m7IzrHX)utw~DDkcDXB3XZp)< zm64b7^&n;zw8oGU0gc&cR*COLFt_W@!*eHsgh{o(AFV&nil-tvX=&)x-eYOV! zE!z?p*%tFgV&dWO%7Tp@9u4~MLvY3Dji<7@%&d*Rzl!9@Z57b78GMQ}R0IXUQbHUn zj+PabCaD(b*cPkW+A2XB#8dRnPZXb%o$fg&)apeN#XaZL>3g2DDbFuyx}lUh`m{X> ztmTh`&i5bt%^fk7bDlNrpS=<%Hnm)zP2HrRol|weSBiU-17K@a5AZGBoDW2HjMN-uN$pC9Tiyf{R+E_(G|H40+q(AP zEud1w-g1W?xbZK_*$0lBvmY+ZC$9JwR!RS{=%dpI7MU-J1&AWi8; z*vQ3yCEYIe8$AgH&a}H|^9-^zZ3hpS^Y;-ZJxLRPubZfQon$eYmn%G|w{&3dp9|Sw#$n617wmwJBgg8!=6&lCn(pT5Utt1m8RRDRLUnx z+dWAFx%te9Z8)vZldT6OJVNmUZM-uZXqXt9BiX^zF z(&EWUGlb+x2!W^xbhOo{)g(W*wZ@!VbrsAw1SV322<=3c!IT({B^M+^o+8weZ`%V@ zYGsq;4Zgk9?EFZB+j9{hm$1Z+klSN}Xcj=!kaH|~+P(jJuE(&~@?|&1V4!SpI2&d~$xEmwGXf*JxJ(=!$T#{*fT~S@ z18sB?4FxUM$puB~XGC~8sBqi(B0vE~-vgCIY3kLh@X6?s#HrizWIe?>`X{|Ake4X= zq>Bs}F#1ccVFt|>GHm=$|LQM4T?1|N&%quaVQk*oK+kj4-7n(^8T5?fIXb^bJ231E zP+ucMAr>yGM_gm3w^2C)s}YB8Y>|aj(X{ z53f~e@#nnMr$$*{$Qg3w0MyC3#BFkNSI_ZzSqg*4lyp(+fGSP%j};acpXaW-;?kdj zENtn!VjV*|%zj>qbiD1)L_JJ%Lp)O)BNv(%}iw4OC z%`5rNdRrtIYnr1f5W}0}p;@9l-!)IF z>rZmO)RhRuo8jiE%IWIp>DmFaVUeyVdn*^YmyJ&w$a>kdLDYtE$47jb6h=A8ZLuNw z&r3CJN)^0fUCcklZlyEyz9yKj#W7-x#h#7gTeCQ=2n3tdCxu}?&w5oDqM>CMdzg0X zNbB{URn#4u2j~pMM=$9BZ@KKZ)fE~;TF7DQye<|Sl%4Bd(kOCqYOt<X7Gr(l0~ z&7baH>@SLh?x{BboZ39dN5NtooS~E^Q3u^71;oi(i zoP<2t9gAe+hEOdA0w1wtVkDmI7!N(E5+1}%QdEg?PEdk_W*ek+@b=$-$RSva1~Dw5 zL4cpjbH-Ba{hMt@M(wP(ETi%*E496k`Ci?@s6tnU9&A9a0rTLf?Tz1eVgG+Zpis3` zaRx_qn*I|zaM?W%rmJ~3J9E3eIwEx(&Ksv!$0{NLsS_STuZ=Lnk2gCN&>{Ds?{jkM zpiE(puY#+_SHcSH92Ck5o9TZf%F$kI=7ptEJe4$xxv-z?HILWg6X|2CGy{Jw-Ys~- z?Q8D6WwD6`MhE(`U~R=6%*iEcOiP=Rs@tfz|Cls0$xR(G)8I2;95Dcd__6>P_4)q3 zqTF3RZev0eA|q|+;t~&ob!^(kj_jtmY>p5Ps4`2WmDJi!+;p>%om=g0VUFN}Ayi|d zMhBTH0$tJC_!=xa*BJGhWgC6oMrOa}{;|XD+3oA}wpz_ z+{D-TBa(|({Ga!m_Q}lpsl7S<-wexQDMF1l;gH)8(X9)?jSzEeY%eB~hU94!7pO`p85_d({2R6H}O(Hr3(&RUq! z4W00T{4Rt(cgRM`jF>~RLMAUE_>KtJDf)f6f4p*hjh^M**7b6A{6w|jwyER&0W9UmtxxAL8pDvUCMmhN+`A6(Y&0gCZ@qNG~zqVOd!er z;s39!*Q<~nVokM5S4wJtGVzwJ+qmhV7zD2oS8+oi9u%Ja7Mc~j!O@>O)7?d`iYE)O zVsz~N66&`pCCDuaE*;)`zFV88sHByg7c`U%1y58{${WK{PO7>FUs}DTGVGamB6bjV zEhUpeC^3<)k$4`Qxki>%Fpf^HaaUG7lgD{yu1W(H)zp>CD>AD;n3G1O8kW90^DZlv$f*g=)68W5rMm_q+J9{~?KQ zS=W~!ip;Iwd~E+xmd?*^Wj}MB5^qMw6X9$Oy^eZ6^`mBeQFfJ~d$?wafm?Ur?>)}i z+h^hMMwx-Dc(hb~$hqd?Z0Y4Vr%&AImCG`GCAbnlP2?JEWYSThO^7BejRvgDO4K1H zSme3CTH=IX^V!X0HYHwN;~9>%8`+XSkpCw#rm^jA@;ZatnOJkve)#6uV-n&!_2D-F zp%W~8{|{3s_?_6yVigM*!O33Saci)V4iqu+klzVpCOa}1+QT)*MU>KxMp_>~C?^{* zv}TW~s8VQ7;Vq6+7c)9WKJFybF{wBNdA^G~v?Ahbz1nzr*!s4AAw#atCx4T=@?PbgrR>)`C}J z9?eLS7Oph3GewY_r)>h{Om9I}$|^3iHI+~Cf&J3+8C17Rqd42tBLlugBpkFGUgD!2 z*rC26NkU99iA;g4{VF~r_^}sLw&1RAJD*(D?8zwMMmD{c{wqH7{@IAJ4;^G1wxys4 zmA2sSHQ9@qYFa~#lmzI%O#q&u6hS)Egnfz0A%c_k!v+QvWO~hNHv=gV#UYF$v&d@C zj!5gp-PW$g)w50#I26$6(+Yc>SviI|4#i3qH0n|JC?V_|S(Z|q@z<$tbh^zp$69#X z{IiQHGP{2)63*8(EW?`t%iBJcm*4 zJV*$;tu;F}ZF#_M$3Cs?nzg@0kFEdn3qoq(!D&PtKxF4ku-@qAzc?1{|8Oh^!KzG< zTfZ1vk>cgXO`D(5F`)Noc6PIYvxbLv?>I^nd%H{8Tki1YnEyjaCYZx z_fKiWs-s*z-d-pc`f?8F5mGRd2o3xfuT?%}kzFnvN_T)VzHJC4_XkuYYQ-5O_Yz}x z?dAOQZD(L)Ad;WVelR;6?fQ*WH!COCheLHVD2pspBT#fWhRT^@fx5N=aoZew;of-8|xB51Z1KLP)Sv2_-% z91tR{1B)1M@Lt4NoJr3K*zJ4<+KAu(YJLA+1sV9_Umxdj4mPij%!2Xt)~@l2{JhY> ztAzY5<3kGGeWx1?mj3bgTmjTydyr2n1QBMS*_X0U+{X1e`(C;$`QfALgu~e{d41YKjRB}HXU=( zI~92m@{8%)QYw***qu|5jilE}ctZ}R2mQ$c9YWRDcYvR!!c~3I8sJt4RR*^DI^gkI z_5r#kK4cTi~> z#6Wm22e2IHtxoNeSCPR7?;QsILAESF^H$>YkB9ZcI6{&bbc3*Oh$sVWMHd%y*4tLU z37S~${J@U%u5miN&w8utdzo(0hd#Cx5I$Mo=zsY+bDw=9=lKT!kwy9k0Leg|$?^x7 zNoEO{!cPV8cgW@rSO5OKvrD=}BFmqxfSk_Zx|;!G8co z@<)y?>env0x=GN+CzUgoOu(xkN`nH4y1}bKp%M_<&7v{KjT^{6=2?N9`uHZYV~COj z87tteTD=0*HiZxxP2hKgu}T+~ho2u!z@GPx3U}GwFy)54}>p2kfL0 zng?v>X5mzBn_2Xe`n|S@hTWdH3aj`nT1IKwl#+L~$HYNBT4-8oy|&%2zHjrrj#7g0 zqrz^PA#H)56mQu>W6YQeH~f)eNJEF(HiP!DAt?21>`#U(<|L{h`_S5OCi~8T>NY;< z@)C-BqVm%A*#JIL^a!>3wn_S*PA?w#!+F6gs1$mE7tND&{cd02MM&X%<)*wme}AgH zcy6yZ7bkDPZRGs!d?Hfr2Am$)^P^kt^c3Ic$M(FJu5rYBWSvj2EgloZHO|0+yp}x( z8B69Cd>{D*Z<_x%xXXURv4Q#;yLFa6S152S?ix5B_))w(eG=0gT7Mai6UUzW1Jai2qyZjb=72w~`i-t>=l+yF`^o*Tp$gswzthycA$tZ0T4}lvlkR+wyz$*U+`&F z4~3SIQkGA;H=>>@Fa3vWa=uA~@OE{m$|9VjqJ%UDn8M-cGZ;Ko4<#aU56SFq3-ya! zYOsz53T&1F4Y*oh;azd};vMWo0VDKTBThUmoXl>kN8*uhKTQ6c_&d;BGAPm~&gk_o zxtwHf7qD*e{h+Z-s!55Wf(>QLMY>UnDz_a;51Rf{uU5MV)&F({a>2)2eHgg zJw*Ylxc>_omuS5r51-DT6-F%TQEqYe5eOUo{x+=Ur&{iyDLm)>9I3C=+q4bg+-; zkOFNs@)7^P@fiF6#$&p^@R%%|F&|34B#7ttfzrh3-T^E`dV|J0e?eRv-m7S${Xsp$L2gWLyL1xsT!d~Dh+Gd3bcY6(sD<1NV=(D!oLSQQi;^} zdl5fhW0YQY^-LxkVueUE(2=%t4Va9HL4Q&iw9%~)Jw=7sL&wZaPmYf~5;kP3E1W2A zxiaFW9fJ;=D7>QsZnH3i6&hV^mAgSvx-*rax-i1}apGv2p&J2aRW@?-Yvv0=K; zq$bDcY#Fpr4OG5F%kpozhJRFkc#{+<|L{i3iUHDJF$hXT1T!}X_U((ot!k@YeSQT@ z-%9xqO=3h18k-b`bg6^~+m!Esw0Rhg*C~Ve&pi`a0tRXV^bGgw6b6)kkTF7*RfE(| zfap_%*}mP*?p33a+JjkZ369UH2^%<}6mjlNoe+u=te4`8vQ;~3tWE>c_Em<%C8$5+ zEAXq#{Jp{2y2afZ`zE-0V8-18W89ZiiQ39yRhE!;1pM5CDQZ9{M_1rXI?{T6LY2!q zmVys|@ivB=>CNboHoj#+7(90?#tLclbR3N6?xd$la7mOpZ<3yj6HLB z1{7ECr|j8o*|zT)nke|rcx7_Mmy08g*|n@oZkl8KLdQ%X3r3-u&K#cpZrCFatL%^;XH`D#*ORd z?)?KsgFS7qByO|U%W0MOc-%qm>&(bzN4SmoPxpEFdh6y~@jzK8lI2s{`2AV#+cM3w z+QU>0(*b0o6&!J%(rmG-A)9O|_%jEc0E6~HUTZgd@kVpWEhSv|o|^Ng6^SPHW|Gw? zw2Xrh7XB1*b#&jVAJMGDsw!oeLt(s!AuZ^H-8s`Fv|hqkzM;q&$WyO5SFD?lhor?P z?kSv-DL{KMCsZ-|;U~&yMIx`kG(2NgR7e@W33o6*r!Q&;_q)ftvb^f>EZS6t0td6v z0`Ow2(dL^xY26{}T<;Wp(fzb^VW*nv(5l%Rlb_k_9)< z?S|>eGujRh%wstS@vovX5o8WdeDA{Oc4APj1t z8US@(TsD=@Ag9lK^!Z{4cnf`73_Z=3Ih<{&&3P(~_A`4TUJW%W?E?8?3ASW>LeU((BL%&O;HVcy>TqLQ(w)N2|8lu1Z~W<1IWau9-cdcVy`Nx> zkk&0lWzRx#YeCxVVp-e<^NY?xc4{8E5ndw{Zj-y4e>%tcAT=d8Y6j&)v8(mub4VjF z#HB#Ab65_P=K}|-+mbZEgc=Q2p};e4zoNGgP9+?8kRf%qs0Pjd%$VWL`E#_GfU__n ze$RUVt@*2()`n3d#f$~41MdN1^fUxTqj8CQ@$g<=Hoh-w3cNQPC|G($zET*0+dkhL zj2a$P6aDr)(X@pB@n*6Po@^x`baY7BOPj_Ii8$F9a)NhglA$-AftZ7RZYB^A*R*g{imWP{n@0OajPK!P>90-_9jN%f z?@YE95f*pabs3oO(1}wE-F&qx8ANj@$+~)SJ(5GC3ewZ`n4Zd1mL1ewoPwZ8((_XG z&~sndkvl-7Z4dO0F-cGc$2#SdWL+v4oPqmVzI-PAr(bchi|@@-Z{W94T#}Nz{x$17 zBgMQk5&9T<2XNF;kaJl(#|<1!{`Tk1C5)!+ARqFa7t;Gn&{iCKeOEsGAw_?_Jr8{w z{I-mP03`PfwKeD(G*>#a>|QydDvhk|uyS45a<4(-5{OQGOa#TZRqS_bbKUWJ>ri(r z2&|B#%SQ-pMAcvJ^lqbZLbz=*-+Nd2R`eO{u{eUC>lhyfGIN04(daG2-aJ}YVvUd) zv@_QI4RjSY4Zex@Us-_#Us{$2Bemz?IDY+T@Lku1KGs8Ej)cZwRTX`%zgj_Zo>{Nw zht6qKl|7F3wQvh(ByOV+BC{8>S4^tZ*YN<-$L7-eJ#nLfg^^HE7x~R^(gUi8QO&Sx zs`IZ;4O8ki*40VUnaNO5De~%u_U>pev-X4j&-Rz%GrgNYJ>)1M9(OOBC@i=m-%eiq zSgl1L0wBEB=J@!2wm*OD+{GTuuB6T%?Au*x-MPBkfH$6yn(TxYVt#3j$PgT@lGt7aI zV-lqILHwu7sM9JYXL8cO`Z3qFO*;2{B+pI3oGo{$>#E zyTcYT`9AF)EoD7h%HdB#QIDfVLL=aEntV?Q2@LtD{6M7k!JoZUOC>2iFi092tDny{ zsZYbO|66v9{eR02=~|D2N3jnbI-cBUM=fCzSqj6+vQ}Ek z)YqHtpZ7+9o)b^hy1JBmU&ilOVkM0c^IBA4ZoW+=J-DUl22twh({&@Yl-~6}+`Q_u zwErKz-Z{9^pxxq)?d;gLCbl*4L=)T2WWpWWn%L&Vwr$(Cjhp$tb8elgTXp~HUA3#; zuKjlJ>h5Q)^=o+Aj!rD?kBz;X$3&fu<%}87^@7E0_v?BU-!KO6l*QuprT;(C0#QpX zspxW)`ro=|5}j*J8u)|}+@z#*lJTyYkk8sBufbmadsD7=Zi!z6nt>-*$x7&GMji)8YX@ z){`+_ix*MzII09AUj2vvi!srMazLSSuTa1ff~rE8sSm?{)rTo7nV6pcqdwFJNh6+( z*HiyTdq@|${r?1Z=b6n03^uf%@bFu>%Gcb-vTTH$<{p*ICtHH7gxbD@2Uyzn;>UjA zwyK1VX^SC3m5fr^)qh+>w2Tgnv+^Y}!PAF~5&-a;PqJM42N(ZH47nb=Vj@B(B6hwu!+M^nIa@u`sSIH)8^J)VxjP3NoTC&hQ#LlxvGitv6=Eu z9^Ibii85ZV_@Cicr+a3K6rs{FNGU3 zSwAZtRRbm{?3Gs2vU9v?*7~+#(gx+iZN5)+lW$P=9f^-sGP99+0~4Y?aR%>c;gPs^ z27i{zv%C7(VQ08DJfp0XdWV_BYVc#&H(f$_A7V6L&B zB7t%&mI=5l-O>yEwg-7OY1eIQ7-rMwbQ>I<(mG7+#0S{u9af6>s{_vuf~LmBG*w)u zDVKn-ih|j*V=nYiB? z25z^pI(r9hz{4FKeR~onV-u}9B3HUuz_~8st&N@s=`eimj34K;*3|-ivGE1(H=YL1<%r`$KG|;VbLWYEa}Nl za`C@X-o<~UJk~LYKKl!S+i1a$r@ZHDquV`*kB>*s$Bpm-Oz;26jjTy;e}$OvZxV{0-LgeR2-E0WSaGe=oTFT zV5xQI&fS#zpRM8stU<(f-cRb4W(JGzWvO0KB|6ZVrWsRz0+DbQ6pof2d&@`Wz_Tk9 zO-_X`ZtP&qwpcE{BjIwQ`@6s^;kS|u0Y#*{kT-fH%)qguYhzJn@czX~$dsTtLC|?S zc6lUh9n07qeu}+aA7HW>^a2!k_8LR`2^P;z%I3d&F|3pm9zGW6<%h3^zVQMZ-_wG^jFqdEY=P9ppbHlsrVdnNb05#WErk>K8yPhs0^tDY`9xcCNdf%u2Y%^4Cb^pni$a+h+3s>)k zy5g!jm<8Rgpdu`~`TKRKqysgUe>?*H<%F~4esic{n>tRwHVPm z%`d`q4~G#@&9l-__)N;rqL+gh0F+>F;g`-8{~6Ck_c46tWVk770;={fsd9&?Qwl27 z4a*K7;8-V-$!3lIuLq;#|SF~GoM`#nuj8Nn{P_j@$fb7m23K7Q?8n9>YYTL6H$ zKXRP^n@N@N4(c)aXD2O8lDiRFbGUn#uo}UY=(Ic65KqnBB#%@TQmAXyxj9Rhr!E!T) zc33J9n_eX0^|4m;ox6TXneP8wI>^NCE~bAgB*q>V&kbU>f-xX0=qbfl8uvQbizg{8 zAG~k}XbDuSO5iHTOgFq3D-&5;#^ejnG$O>s`6pxes4r+1!lx%H-%DWxQ(v#rf z$Xt>rBBTokg7*c3$22YG9$Oz#hy9?Ze4K!Zzq>mm4azb?!3ZR|TtG(~9g8}s{HdQfWiwaDqsz+_=>)pE6q@zaN^}d^7^kU0OyboS ztsTGpZL*VpiEf8AYTfr-$wv4x{AbUfJ@-h;TKYQ*sqz zsGX8#k}*zVaBb4RWuKVh%IoN!3}t=l3F`Ub4$9>Z-yAJD1%*Jbt z?Y}UdSe&+Rm-YaL|G*bqyyh>rIL9*_U20a-1-Y`&ce%xnfH-E}kYiBJQLn+$)~dT@ z9Q6wGJUY#;ApBXEw8o_fSN9T}6Rtodvjw(rRdR%*7nJ!9dz8XN8WQ0~*`Jitk7$;@ zw#20Jt^HDm!?|UWFi9=a0cahcur6n>THa~d1UM|%j?3Ma@s$BaHRx|OG40US4-3x_ zLd#5gZwgs-vmsKoFK|ik=2PtCqd+sZ9D1l#e(JmCLr8j*K}52_xx-Q)kve<#nXFk0 zQIDaU69XiHph7T|+C0=FdH9yZaQ!~mcUSp=u!@TPR3?~ZvzFp-@~6xFM1Rc$^D=dk zm+^%=Ky-NPy;?ZHa9JeVF1&u(kS8iIF!N3YtrM}z0V71v1YvKqahqpuELJe|m96$P z$+EJU_)p-fs=mS4n;YaVSrnG+RwkrAr9qx-O`4YH|GskEnsK@s^%$vwPC{VeNkfwt z4=W_ruE!*l1a51(lKM7H@Y#P!Vol_^V3{85ay;J3tGPfCi9)lf_{-QVi6t5BVEti$_LTX}S0dvh&b zo@+avs$X83zy1-vFQz>wG_Sa#0;ZZjh39tt)}L9u_U&!4n{{?YEFGA?i8q~=J1^=g zQPZx>{IuC)ly1B!gZ+`ev%b}XDNs~N;Q6RdH;<0@mcI9R{ftFD_JozA?%lI=pm~o{ zyMf)zfASW}+OE0w8aV{F{-UatC#zghFG=uMTSmIhQ3pxy_x{UjW?uA6xpMW#mBuv+ zUQxYz_^7W>@WH)u6f8*jnNVU&Jc0&paVq)=}?ao@v6 zc4cv(n<eacOmK%0)F1bs!T0RB5XB#Vf`mUX@mt1A&Q(QUa*zb~gB7w-Q!{Y<`n zIsMQoc*4jmnvfC-z92tCzrE2#)CxFo)Bu$0O3Mzr3?R2T_{2$SoMO`oulm;N6#F(m za;y23_}L_)k^wh&jyp-%NIL2|P20Hw&hrd6q)FWD!#_f`8)+~LbLn@>(7Y_6F>Lz4HlrM_)9ymp&D{&-jK_#j~U1yARV0x#kJw0lsY zUL4&0w)ekhNNvD388OWV?YuUy2%XS17T%d)n`i&ygwKtH4~~80RRV4a9DY2VW4l?7 z!n>s`?P(@k@aKuEjr-T3z)qvR8a_QTG_52SK+bq9z&#Dt<+jzSZ--0HD&TM07w#$R ztoMueWWLh*pPb3+-a+iff6s`9QDLhc)7Cw&D2OP+Q`RH#Xy`feB@p7Yz8EA?`|(Cm zC2k*JC;x})G!nxVU-ADM)P{XHv>-Wh^E#qMJQf0pSrP}YKPZr%r_ZY(KVRQO;~Tc_ z7`rg@s#yBU2Qg}OZ$OTNLRw;5U{-v_EPbA*m{t&$%h*tzw0^4yPs$Z z!dQcAOA`uQ6Fz%ei#!-S6@Wt4dW`<4TUf2UlOuwMEeb$%P_2KM(pgC`m0Yml z3(og-L?L+IZP6j4Fq>#P5H0F|3F*P3hq5NE?zYO2W)gf1}ZXH*6aR?o6t&g3mLKx8f)Dj zvr38zzTg7#Jra;Uv0G5$wqc3uTUNz9MaRkIySA16Oe=K)zHf&rEO#}RiByqq>1KKroUT$~wBMHmZ}8ATKy)pf|lm%=4r%oV@u?0u08|O&6}Jo=s+fJP>KT3r>w^ zt~W%2*q$Uw!O@|oR{OoH8Ys^mcDv32*#I{9an*FLi`WI zSsk&##T_@8575axIkG}&+Y+5{F(xSipVjw}2o0;Ut3YX-1oZdNR6m~xx3>9rarW){ za>@|w+7j3fu4QknN)mDqLojH|2^TSe`aXyWS?X%mGF{RyD$;@a2&8vW@p0+aI0Ul* zz>d%pw(D7jaI;pqZy5!rhNO^Xi`6DNmpNe_2XgqQRh85s!`+fue zV~`7&y{fyq)pz|$>)BzZPc$6ZUA;qS8H8~D`0|8|o;7|d5K#3Jo`0r|H4v>JYuw=q z;;j;N5OgF0(FEYc2dxYe4C;O)41Z18m9g;Nx-W~)_KYySks`mU#ew}z z!A~SWa3j|lg)=fgn%k$BjW+u1nG;Xz)dPrZZGG9K%%MnrA-;ofjFXv}mlTiNbj>l6 z>o=wNSo}~>V`oGO2PNXx_c+a2j(FI_&A{iJMbV1Y)-Rmj{A7R$op1 zv4x6`&^#nj2LH(m6O1kuV;4ljgwJnP1DXz4v{A?IG3Bvj(I7TA)x5BTz>Msj>c|^( z+-mg7rGZL#%VhBWXW#b7g>8DI3%>%v-2*tm;GOQEA@52-}5m?0|* zl$JQ+J8|rr%xB{e8gK_h@EIe8k8shTYlBl`Q?abn#Ljs`m4ww6Nd39%;v$`Y0-bf7JOht(QoFZn?=l;IIuzSOJ zO&=Cp`P}{Sd>RMqTvlrlpl1-%fwgW~x3Ff~%CC!{?+H1&ogrTz5uSI8^vr@TB`gZ# zRFtJrFUCV@t>02)=Vd^jjU(o)M;riXA42sS^u1{ri^ned>2*bn`>oo8tY*ktGxKG| z+Mkumfv@j0Y7TPLi%-}Rw($y1WYD7=aEGsRar@noV>~>Pbe=KV#wqZ`GQ5!enbiLr z=*+yu-$*^!#U+d~|-MD;FyrEO%1Xj?5YLW|<&x8vb4|L=N zJI~^gcHXTBSmXrNLG{Fu_KyHs82b>X))(57;J|B* z7T6W|z9k>#$Tp-a9n(&mSu!q7{OTAWV%@9sFZ$sp+K;ZD!tOpR3V~RS2tL?P%%j}Y z?K=B>ZAR$ECnC#N`Tgl(tL2edLaeR%J4=u+kqD786ev=lnmGMhDflWV5`#cKt}RyQ z{6@Pj$l68fBy6J~bg~cdPVU)httuyLY_Olew{wAN3pd$HKa*mnOZ&wQo0-T$|07PJ z%cx0H0CW?dyPVtc=cIV=7x@m&jmwGNT!!R|)jAA|^jUCcbYVfkR1)OvRHgF!V|8q= zP%?@x-lA9&-s+x7zhLNQ&6}t*f_K&Qf3q^Vvo*23|F~$THsuV}B|TiB<5r8slbM{^ z$%3C+szNNf)d6+jbK+|x;rAl{DOb>7?6y{@M0~R?q2Bym1TcBdRKH;Q5LZ>?Hm$}c z{6P~m8S`ex@|BSYyt{3i*oFHqF70hRPNwWTCiHE!jp4K;14JyZm_)Wo8PeszL#yt=2dHz+=G3qb{2XDzJ+ z0R1^b-s*914y@OH+KVr)p8d<9$E{$BN#B{g~)7;#{v zFaDfN*!LK;MAs!PZcU;KLctWv`Idnl6T>j$RM$Pia^?lV8(aBxr973z2PZEC-R z{~#0*U)!4pr?!{8a_SqrkwVrLvGnNi4kMQ|_5Ils3TD5NP*AK~ z!6*Nn0N>#Co?=IAu2xV`lvSHfI$$&aMd&X}AffJvA$GDQz(TG@wBhP*T)KZoSlbLP zyB&Q#3My{r`~ms5wMXG`<)q zN92UoawK#REHQ1mNvI$eP^lw7$rGx%Ge(!t7Dz&aN1Xs{-LaqMe!E%}Tw??NPm#AW zau$6SYz5cE-**@L*2N}Fh5kdk1F4AAr{+Mrj#gAXs_%X@mF=rdM8bKxKO3xq?6lWu zkpjYe`Nc%fvkfhfXfX{^s~<^|qxINtb5fAUf#wIItVGdeE4AN<)KKoViD=%C|Ki;m z2&0!{!_eVG(RTY%0}+pa>K62SEp>f#Qu!h@BBAn%8;~ioxl)|s1+uGt+3CJn4ac&-ZOSMTvD;uin(Hp+7I4HfRFk8|f+ zs73$!nb3-hPN17OM*RkY?%?}}A`SeSl_;6IZ)y+;2dtx4t*=VwR8^D};@8vMPY}_n z_8&={QFk(E{cx5@*=X6^K5c}l;MC$CKLl}GGY39SFSci@bD#*{sHrKsuppNhM7ws8n7F+v4*%?tLu0%L*P?`cA7GIu^2{$+a1XPIYUv(K!~ zsyL$#*5Hy#eeIRKbMGFMHsH9*i_fA(j29yB3i} zwPLsmGJ-2i``}LJNnWA)J);r8*W>$HqZ53dTMiy?!*}Am8_6LvWTl{9T|giEi|Kal zK%ruPAu`vETtc^;fYi&9I`~vDsA425k$0IVD%Y(>v@QF@99Jz$wnO1SN&?a%m-6md zL8o`u@8!&xgmGW=!cOp9L(!Av34KME=C&>9${04zDn`-8IZGbt*d$&C(CA4zl%i34 zJ4^8lugt;37Gv=YZ1tpw5|Xv)2l-@|GD^SBorX$;NRy?{AsyOAo8A%IY!J*G-RmixXdMi#>Vt< zh<}5fswqoqy0qW4&)kCt(H4%;q+g9g6n9cj1*t|IeSWvc9Avi%Uh5(k6Ygp-sZcz3 z4Io|YcZQC*SkQgEMQt40(Cl*P2)g`*ZN>!Nc_~2Ug|&@Pc>&vR)MO|^?0=ml65*>* zi=$En>)kv~A$889C+dc{&;Dmo(QAW)0e0l}(i(31*WYWPqa7a%%Yz7irY51GCDFM8T$5>~@ z6>%oA;;H^&$D7`71-TMAT7fBWZd6^c$+c>Rb|o6E;{p|SNykDMQ*P7H0}1-b}*#r?J;*|V7h%IXSq!Lm8k1}5LSLG3{idWMYBw$RQ?8rhQ< z^>zK9?2Tez!6#X+@M$58bV}rMXJh0#xb6 z|K2vx;n#@v-?aO(Mz{|BWLnTBA@D&SP=;*qfDG^-nR>G^NvNemu@+?7vvK^F zeD6BmLUMHl^THOa6)ujf@7j?`tkXwvt5$ZBWr!@OeGp7!dH+`Bq;6ypS-_rHf$jXS zS@`*)YH%#_$riML-O`YPD|%R4!4TA88cwK^wqgY``C{k>+x1L6?V!N6req$Ou7&Ts z1w2)xb`OR9%)O{M}dHg&Z2D~ zQ^Kkh&2s{m>vXZV7GF_ckyO({il?#R7O?DO8Hn|%s;uWSoZ!jB$Ir|EsjZxs!&II% zd&Y+qi8t{^0=&_|3tl5LfF{0_v-Q)y@B>IJsd~h%errMTi1TMyI;CtlT*+C>Th3wv zuNN#6zgS9Efj!j8xAhO=|@0YfEIgHxB)_w&%m!GypT+ zldM4WL~uU(0u?r+&B1Ki1z_iPNvN#p5P5Th3*$KQUwDaii)0=uKUj)YF@V`aW3M9c zN`eBTpQ6@$m64ky;~n;XP;JF-im!iw!0=MRAd-bkkOWl<XSStaoezohYk zO_RPz?b)zl^+)$>m9RNKdk z?NTlp+Tx`#965H{K1Bsd`8@1sxe;9$YA>a-8kw0w5`8$je{kEd z0YS)|2O3g{%5yse@eb2#g2?vv(rX4N-&oGEKS6SppDUYQc5lH#Ft<&a`zo$1h@0r; zT8wE@ol2A&icNo_nYXDzg-4GxnCtiq7V$Twylgj~hU*~-dc8hCS_j*|yJP>ecwF#5 zkw9<(sPdscWU7SqpsqGGx?psAkAcYS=@od+h(+w_3-<2k+fD?9yVUlq;G?i$0fIhn zA!O@wkM4fqN@DL{N5Sc2XyIFEf5h#`b(D>zRv&&Zs~)Qn6yokbJR8i&pDIhT*&NSt z>s9cJC;Ownq%J3L>cr0kHzyRBCbqB6tlg>TtXEB%cnaDRggDeS3+(OzRg~GQx8rMy z@MDOV@q`ck-WMOrWIrSI^}zkuF_qA`WU)}V0>sK}sz0PG!EjlbU|GHK4NfVyrbv{~ zua+c6rW}b_>&EiAt#3oH$PnJ0inEhEaDv;R{?$N4ce0hj600cIEd&xD;dG z8$%RrS95RrFHm5r_oD}wYU@qj+E8>Xa612dt#i*T?(Y*Ullwrg%x+A4M5-M0&)NIG zcwFr$)oZ7WCx5kZ&b_on!~rgzk%C&6?3aq1XQ?hzZT=jONy)*+hdLfBIrc6@JC{>I zDBg!T%bHW`DmrUa#9X{#Avimium}>ech3{>lj7oKLkzij($hmbq+(sw5cgDM9Zid% zUS$({rUkvh8p%*BmA_U<&U&md&|uZ6zAtpwa?eHPskSv30SO$sJl_(M zu&|F3jCt3VPNl)~E{Mk5bx^wm1y(lV#UXQ(9vn`UP@fB8`Yr!lSF`OByZu6!hTc+V zHvDUxCl8$8R41!=(=Mndkp_kd?b=%(V zP~+08-k%*&XBPrdhuw(*@##cZ{al~HORuL2J^v>fU6>A}Cf(*s?y5m8x3H~;pSIT` zyw+47ee-#4&hTgb@TGyrT8eTA^#!{-;bO9F4O$Tp{W`%ZTIy7=vZXz!Y-iv-2`ITKOt zD~o8|Az~^1`mP1`H_mMVo*)cXkq_w&CGe5iUi+aDxk1!6;27^(v?8bzQX9iMyU|m&4G|7J*l$;f$UQQYXd~H(?}D8NM8akC$a!1lgkR5fVsYc< z7LtEy$qUDL0?g&{o_D~{c;eernoDZu^jqYC-AodjH`zOzcdgjRz(r*Wmn5HEwp->d;_U^pkqg342qkO> zbWu2T!8Qj^V}GVrU45oI9X?7`?MS(Eu1Ztuu*p(SCVKyx<+QAt>2%4C;MCiHTN*d> ziQXkVtBT$JB`v^l6ibUt25eIYXJ5cU`c;Yu)VKpI!l=6G=re}U0^)%JW^J!4=o2p? zdOTNVpd6H)9E;{D;-JhVTZ`xMk#pzjiU!PdW@tw4mNP#TY$eL1C3q}_#W&y%BMl#% z*xRBoBMQ751Q7XU!amvcTA>|Ng8X|rX0Fy|b@e*U38^u9rZC8oAy+os@d*mZZ4=-p z)PU#0mN(v_A+63UC_a7{R+b{mDo5dJYKX>Cp_gJ&X;m3J2Akz#F~6sDp7U`dir7N- zJEZS}VGNJ+Tix;}EJJ``mjJp3VDTE?wDI|mF*oP0OlZ)WncbUaN?%2Tp%R2XroqKe zSaqQ*m-u?(%80HH#$vKRj{ETPE0ZH~=HN64ph6>rDmT#cnfP&a`7938Ylp%sXUH+S z%hxaRlhI6Gfw0j7*x4~WZx8++C8eIwZ^ z%`DQ#M{hwj%0le7eGfg($4$$5k1JO8C)3 zY6M~FZj2PR*2A?D`~|gO4cJkVTfMLO@0N}Mm0Axi)3-CZx?tQr9hQKl5Q5YZzEq;& zcQ(P?XVmF-oPt7}_f$1jvZ6zEs-#7maX)iRv0CD?em`n{eR^NWHFS9KO}(!amXD*T@_wR9tL!iC$l|zVIfx1 z?+t#Geq{&$@n!)<{y2*eVk#u|X!<9aaLfnz>JD`E%*8<&_c4bu?#D){mkc#gBp!fM zA|71oAI%rqu!Iv+CQb$W>JjkOBVY)u=RUkZ$oj0o=1c$-((u;5LdbV%i9wn8M+aXC zDyP#sH45G>486lEJLFe?a9{laLJI$_XII-3}Z9;Iz0GNjE$C=uOS;+-N z-c_SbE=tfPKYjCoNmy}`efaWb*w6Qh+S|d-9^xCc z!03!94Ha7U2!6CBq1Mg0r~8}>&d%&sq-+wXi)HMkAjoQ$MTI0Ps?XPaaH8hLaq?0z zG#-(FIoy{hPJn|*q-|0#8Ub&GR#2`cG_e~;5>v?g*X#=}f$syIXJ+3NMN~cw(&<$} zRCalIWK$dmpZ|AXPrvKL{o>IJ5VP<5<(P2D^<}aF3#2B)ljod|D)_?j-#H)ltltzM zRwpv`^^ZP6A^7@Nh$K(|1RL|dQq}IH`OL;{W>MuQY_CK^V4z@v z3JG3(X|j&T<)Ai2_#W)~!_P=ih>l>Co=(tt5LiIr-H7SOT?>b-Jg`@nOl`{4XqkpJ zQrSxy=JcR!ay8J$Q_UB?>PG)*Yy?WV86caQuIe`&7r(xYTiUS%sgtDc3Cb#uC1MU7(# z)bv;Ij2=k4`wvys#Bb*xRrLepBS0Up)BUPzBxWIm)X+c~FFn|Dxf^3xM$pHZxVav+ zAKh5hIBx}fif1JRgZ+!6mkM#-gs7#o^)f`MwW_l_o@ik0`F4J^ck{Am^m?{)EB<=B zKWulNS8GRgUZyADGIUp;wjKgm1PLsULztso|IX z7+b#3@S0+b;E*y12JMi zCJY@C+v~%5rv6hYgv3ACs6Ym|zmT;zYf)Uu>i3ZpDTzea&Fy3^u|-h?3L%m;Z34Ze z=Gf~Fk+R)CtQN)l$K)w(4}6Wcf~!dSY7j%H!q(P|Hx{Jr`eEtW+vpDld?4w18;3-d z2t#{*{?T`bRp2ZC$ar~iA#D9sx7Vx&vh{jPS|9_UqY9cr9VE zJ&(}x*iD=!XdIc}YY^9%O73{2cvWiA_EKbL^40zvi!$sE-$UB|6LCj%p3$=?H6-CZ zqnL=!wSb5eUJ!}DI-uJ;Ep0Y?&rrEHkmqq-N@9mXxI0Mv_)i-nK_mQb;FmH zt8$egDRL$mC8NL}Xsr~pfqTvYM4AS1gITe0qcB9i*O_Nmb6%qPfj{~fcIVazxf(R> zci!HPiA^!U%YA}Wd(q?Fl@s2Jwwc0sbv9x-fDvUAOzS?2?Vjw&!_1`45kvwfvDk6> zgKCKA7;Fqslb9pxFX>5J0RsEkJpGg8w|p}l;a(F)9$h?L{g$l)=tYA`pL$?_t7=>t zc4SJ7j!zJmtw)JoADk!SLy|r&&U1VgOU<*hraVS&#hlKlnfUo1X4vXN9o%1HoL{yJ zZD}cc=KWQYC0mPL6AMav3>}weL-Ep1fxo{M`T51q&ZEvo&imK3GVr#ApZ3wWfyaBvedY32RJ;~z zhHodkT;*N)1pnPQm9|4_xxc(jArGk1GMHH)7fwgSpF^uO0!>Lm$;#5QXfCXPR^G?j zfpd>rJsR8h*5Bw*d&T2YO@c9`XG6KjX7O?NlhrPE)ymqop(*S~+*m2ct3{||+Rjz{ zHCfmXtu$TW_MBDMtGDpPv+tD33ytzOUp5J21py23tnecdJ{h=8*)kc418{6)+G`RX zz4Hfm@6pUf+sZv?6GRWG5;tV_9Ss@RN`h1ojLY62l>;dC6*__4OY-O06W58m%D5;! zuOoOu`>@{0P4ujLWNwVEHJ6z2P5jD{kH2r%03Lt0Kw&kNH-66AN-V*cKk&D19vg?F z;dXl4N1Tbn=H1Tl&h8tN!JX-u-V-h%o_)}?q@~frML&cAivlBa7_&n7Mtc^BeS_0N zV-^^R6?Qd4d9hO@eTD=hiQ?KU!J*zK8X3c?d^B<;Y6Y4h-;<2Ac6^Bpt3OSu?f^>CB8!h$9>cnEA8qj;wR>!D;pAkyG|mwL;WGDPK^Y zpZZHoCALO0;2<@$C+?`Mhxhs?)~&jiPd!PS*bh~vWj5M=KorP}W(M0l>a$?-ZYEHSM@E&94f!kVU?PgA&~*z*kG!n+kLtSGLGJslQP{ zrpOIBcKuf!ez3UR`>}mZTP^E2yD?cF`!FJ%UvJ_Ir^hFGi=A3M9T9124X4}n%%fp4 zUkuI6UemevoSeG;-O`_x{gO>dx%@ji`$#%nq#?V~Eon$9kASekLxCd495S1F-q5pS zr|bTC6NiB6-duXd!5N!bBi{rgNgC}kHKd*Rd$!m?y?$%!Gt7uZu64b;ritpLJbZ`s zNz_ZXb(o%2#T`3%s0+~XzWOjD!;5=g&sr-gz5s`R{0O6_6CiQ9f~?;P?`g)>7a)LI zT_lRE^85>6VKCILkZAP5pP9StEC846*Rhrr*iruA{n=-Gv1ux8b#~C{FGz3eHM|TRP$I@5J)uZ0tkWKe?w|VPc zz{n-5+ecIXz^|iv9QKkaU_u9nR`gYFUwU<#E^d219`1gWcgmC{sem=#mLg@${?yX!6)dIlWkRjk)9K|E^!CuRNcl zp7r#jr7otgz{yi=y4kMa^!ZZaNP*U|92WEgNXQBMl~6Yy;H^Ct4=ZL94{Be#bXj*S zzu_|IT+P^c*T8vkbef|;xvQrNqP#M*T~S7V3Y@AZhw+pN`GgDfF(Y|30+N|~paozr z`&AN(W%e7LedlYqyh0FE5?lf>7vm(6Y}h{F&J_*q1Jo)}50Cd>H>~=)s1-_yGKsLb zp6K-%$hGN;ywiAJJ#;*NS)%)T)>7j2$t^{kqB8^_XXhGX5pPRx@ysD8N~OW3=&luW zAR%F2*mgr%*MT5(1F>rC8~6-HMpv~nR1&A6#wGk- zLugQMt*z{qD{N>Q^PD8<**IYmC*=PVDM+h4o1p{$HwWlrzd~<DtDC5M}jLtO>q)_3p)tv>`#NBz| zt;Hn5aNOm^C7r#75|~VRPgx%t;!QDrw0F>pjqr_4@Ifw5C3 zLtEhH_sBP-i+Qf_q5y`Kd9!MPaPbQ0%ziwZ`)xeIq{(H(#m68NiOfS+(I>)d8JeT7 zAPP8j?gv9nuKAeGopJ-s!)3P!YBxQ6uIeER69&4a<3cz2(hYwae%FBDhZ3`TP_HtMu3;BYvC1nO%?Z?`or`d-L+P`qlQ9onlhES$aCk z6r99+OvkUbLw8K=LRLhd<3yjA*sis{Y%M@H4&8V8^RhHnDmnrN9*@r@3_TQhuBRUe zFXFr%?^TaB;l)NNA)6+aa`dfB#r@B+IR}p>(x0{wkzgN{6yvbcK_fgx4B%r<55ys^ zF;NzaNX^X!Ngua@?VsS|nP9$1WD;HgG{X5ga2`h~{355miJxNK*Ytu2%hdQEV1~UW z6pEX{4#ttL1QpTN8*B+{ED|mRLYTj89hz-qK|o1-laDx#DLraH<-9?(BNdIM1tc8P z)gu(?3Fr@JtDLPrsIC~l9^?$TiK=d}KQ>^s^&-{WTLB#E=aG(npDKsx$w*SGnU)g& z!}Y^C5lwO=YWJYnIK$CDZpM|IK+H0=L#11uqS&TKr=qWr4O+mbJDg5Hfa@xk1&IK@ zA=5;nEBJBh{Ooo@SNq0!5%`9z-*)b-TA@&fl0v5n9~%x6tLG__-VwfiwR+#r6in3V z`3fd{^r*wPFtax{vj@O5SAdV#Oa?UHMUKI34t>c(sK0j7Ao8YgQ$TacJelHOkNuyA zkq*k@qc<59D!u)8;}wbysgjr-j)tPC5z=|Bf2}?`MLnokuD%qjwWy|wyq%rqX%fwo zCJB_PKNd&X)Q%IMv__f0fupm+WN76gpRFknEhz-&cM^OpNDwR-u!0pNnT1`Hg}Xc3 znrl<|XlRivpgbY~_egznujrUGE}||MfQL2>wwrSyGprZPP|e$va~sJk79~Uw$qhB= zCOA(Iv%H*(q(BZGDUz)4g+ z-he7m`#_@_X{rvuW7caXQlOGpW9{Q*%tj6&kuZ(abZy2uuX6UAmZ$S+t=8U*+2cv( z`j_ncvd8#-H~el)<#PR`<$wEvqMcQvoB}b_Qw0f%^h|l8fla?BOw`-=h<%0Rutgdk zO6D*RwxHD^8N2I{!1S0|M&Af~M>fLVrjOEm;mJqbko^q|@B_Men7Y2J|0prxRFcih zTfEPYCZ^-a^<FVHLTCr zyIt+jEEv8D{p?(k&ATMtLe2evFDv<%htGK*)HYp8&!=GJjBIhc#gSpuAk4i%+_y<~Hoaz*pTHBj#rt*L&1`$Y%x{Pm0%ziC zi&G<>Zm!N_(8{2xzGU=uBy@N#)(M*fP5q!J-r-!2F>3;xmsm5-G`>#pG_I?^u}@;< zpTMP6W@kR-Az7`ST;{&b-9eI%j^>mVa^Q zcoV?X-2&6JAN_${+YVTc6ccxe*kBJoNGrkag3+D10?sN`e(FTY9sYNlpBSk@r}bn| zcrjR*n`WPdUSalZe^ulZNmnR`+f~q3x>MR*J1ggv?2=TE2cS4#F4V)6^;8(${%vB# z4P@bdGve5fTgP8r8_EiAj;4=4MPWcT04JIjT>*4l>TFM=Lt1LcDg8iK)0`B`ie`EM zs^I(ZG{{@n{xz}wf5>{L=t#qcX*cG?w(U$Xv2EM7IdLYo?TKyM=EUa2>Dbvl@B8g< z{r}#leefLgdhXSISJhQT@KB--r)*SdqS|$s&MiZTL(p#WIYu!27p3;QYgj`iI|(ym z)FQ_u3Um)l<-(smMzEE7&l&P1NQK1&MnaN>Vd;=E!DSc&cn(Pl+?(fze?`a6`pOj0 zH}vC5+Us2vq`RDcn|@C->sx6O52K%bf78I)IsKYEJmwLtRpA5{)Xf{U?V@0=jLrg3l z#xJ>H43Wj@iu60p-0p;N2W*ZL`znbyG#=BPR6Dm&BqG2mOjr`63k z7%00~)+ho}$M$GhBHx+O;4o1WsD}>HPiMwcD4%9^KDiqKsqZzXe?QkFjvAGRM0?QM zVuwNHF8&l|Q$YR#xZy;MOzQ^|Ja&(m`{-d()rUWm7xc!P^v=;0 z$9zj`o1ShgbLONV`~z4Zne+7$nHdRz(lDFG$I$H(P!ZoNYq0whZ${GXi!%bM?=)s~KwQr-c zBq&3a>R-fXR}Ud3bM7{M3;{wJDbc0$9oop#)8e)#Zc%1gvV0C3McM#<^7GUv;wOo* z&oUJu&W+hw)fLk70cig5e~UPwD|_!+CAI%D!7@6*;ELN7=joJ zwGbQLN-MK_l%>qXYxh#nGw{qAc6*VHyi_j<2}H^WDe$vbhjq|0 zzb0PKG_=}6Cva(P%5*)AXjRpvb8u)3pGPmdh)s9Sv>JA+2|iu<*!(vya{-FJx_oOAG5HSRSr z$1CKd&7bPc!jEV|_ZDqsGj2IM%6NEL`JkOp7k~a(_2qKtgFg=Ud)4+o=IDzQ-Af3a z#@VWU+!Jq=fuB+gJNpywKc^-abCYDrlCbLy$h#e`^=&P*g&PkzD%bAq>^uHSm`3~n z`Yg{iF(PK@eTT%SJ1T3p+QHAty9GxD&z!hJm2fM>`oW$E`|wP%!BYYm2#$7!^zcR$r66{vo;bZK$(qE13B`e@-DP0zRD7)-Wu=;ei zId*XiD^u-Nm4Q`hMOdF91=c;^*m7fc7J&)jQ{e^LgVWZy;8FyHvXl@T!D`TEs3qXb zoL|Er0)9pXz68!mN`_FDho*SK9|*O?9R#t)qtjv#eZj_=HHoc-A;d)r!UWgbv$`1LGfT|I{p*csn z<+FIa>q8!k*94D^LSTM%Q?ouOl?@s5zCYqTh9-w?UZ*ZN{;_#r*0igwVpD>8#^p9a z-*$na57Z8uOc}W_bkqQaYCQ^5tB-G`6O%{0Z>bE>xBBex4CksQX-3{c{8O-;R9lr= zu-2oUuI9j}Kj4 zdN_IcIA?sk2+wV3YtI@VT74^J(bYN}-)m0nn_XZ8%mKP1v1-L<1jnaJC9hc}5bL+Z zTA)MCkq04sBt(XCFi=iBB4BM4cRb3`lEoZYg-O(3B8{#5Br)Ox6ezyqSdTX5=#0ij zf{h8Jgu4QHQB~Vjf({q-^kO0{PT_qz8RHlydUQH-1J(?52Q`_`a@VgN|LTi>L&QfV zGfdof3V3%x%0W-t=Sf$=J%l657Dgw@&$x;|kL~b^Z z`h4}kz3R$35(S>YR;yV*MYX1FQSgHB<29(y^lM^7N`mxjs821eh>wY8+WU68k6`u| zb_USQR7teFxKw2SLSk^_kYCxshH~m%!VRQP9~xC=x5;|-$JKCI4Ya({&U|qRoM}C8 z&NU}G%57JhO_)fBYfCotAL9PU@2Sh^$| z*aG>LPC9zt_dnM>%v;`?9?nMxmN{xi7Erv-bYr~NrvTm-2_^hvo5UO~UKX52o%Y9& zNV(R;n0eD^N|WXA=y`FW+6Sd0eTV&r*#qs-@Pz<;TRS<*q}YG^Im7BaH?Rbu={_DjsRG*r z+v>G|J%t%h81g1_A1l!PMy&D2S964(&Twk++}0bEY{_08dYyI za6>1jsn+jh6S%FW4<0YG^Mp%uPzVI=ufGnyO=&0%|H+$eqlKr^d^SD7p0C&0+m-O1YFPpq{v{b9DtYARljaeQ_C;5nHEw*-6i=o9 zzfP$)J$(H<@x4CVnE0ODdRasM984&hXth$2D$VdgpBOMn7;-)DNI)v%QmnR5nX8V0 zXC;oD@)$GpFjp{PW=d+GEC%8}Rm3X$VzZU2roiL8?*w#A2Y4nG5h%38Qq7gGGGF*Wyw5`Kn~y7f~8X>4!q9nOeS z*&C^!Gd##2Ha%b89FHkKb*tm)w$kWQbn3h51mblcdg_n-jyI~`MSI_}vF+|f{DqxY zMZGXGi^boYlxeHk>>FJI1dNknc5g7l>{f_c01S`Rr|&V(+o117(~abJMg7-56j^g! zE+2tF{%Lus=5z={%e?*I2{J-_SpHUO+P6o=JP)1vkC>&IiZyqe7ybEGhyT-#^ynIw z_fB`i`UFSR!%xL7ca7KxkACHcgI^Nd5>BnDLWGSNNe?*J84D@7?guEn>v04c;z9=S z;_>T$IJ*OI-ov3#LNtTqCx==@Fvynv8JL-r{)I64t~tgnt_tW4@+3A|3sw+ggA0#} z9z2@(&7i0nj0mWZw)Y}}fLp9wp#t8j@zgaNz6+E4~c4HF9xyAc=z)oRyR+w(ge2o-*Ip;u5IZ|Zg)cZ4CUs{}QE3H+u*RZutS~1;9 z-|cb6HEyNQpHqS(ge~YnMU0$6hQ_?jGk14@hz1DI9JzcLGp;b50DR6g+v4STtwG1U zsHpGpYc=cmkQ|Y6B|Oa#l@w1mJ)}J&3nL|*1E1QNj8Eo zoy-`8biOF_7Fqp%%1qFY3AwgabdV6?LwI0Y((x#>xP|4#+f_NN4)u+kTI|c)Ptk@c z8#4fhz-YJG%_nVz3W>zWMGt}sVWG~B_DP$z5>BvSSbS)P3dbd`QQ%y`B#uPTswZ0- ztPx@i&6rOKPXA+^-UVRJ;2CL0eGSyo$DB#dq`@!9+FxZ!ZgWUxLudUH1meEH~Rpx3?>-GML7 z#oG3f{#w5kawO{#vu>d?$HRd`ro#5ffz^z2(l?+wX>u}Z(%*->i=THx*(bs}0_4_U zB5K4pgn2QXm#Wp~Oo8jGG~3_nsOQuDh1$6*ixz)up@z$o&v`!Cw*2|e^9A_B-ZRX* z4*oW`Cm07Q*)Z&>C+7~ZjkNWGp>s$6_Uk|Xv9;IN;*Q+nK+T&%2&|^tu4g=@TL@rc zsxf134iI`beQaIe4ayzRs97llaY(;-#w7O>p(o`Y_j}R#Gs4~L{_Bk_K>uZ+d1V(* zt+qh$L#>S0kE;AA1Nn&qKTp^9!pox6K>nr8-h&;gmgz0U_=e`kPwwQNNAO*E_|Vls zl9h$91G2(8enXvD@4@j5f~PwTRvYSzG{PCu%WE4e1DQ7)Qc`QHuVEi)hRadlC;lnF zKXA;>+sU8J3Vz{|7n=DpocH=@(lh%+>dp!x`X_a1F*&+ya{pJ(dp@#|0U8RmGX z&J@F&r!AIGC5LKp0^xDS^0KZ9CdM0BVe%kZc6vY;Yi|D#WR(efcke^B!SWs+D9`vB zC!e)>{v+#FZf16BV=dvL$3CGq?XiqK*{M5zpzZrXd!Zt*;AX<-@u1SVle6%3Rn%%Y zfs7r;57lrE-44kUCjeyzSHuiaBvIqv+w=J`T4UgDql6o9Dd7-ki$}_yf9O*W(?P3X zV6wT6*BX7*W_A&y!*kJFwES6SW*etn%Ox{wZ^~CC)M-1{` z?$-q|9ZgCrlPqVb!mAhJdM2(bwj2zW!DbznZLt2o%wgP)agt%wsrOqjg-dg+9^J4B$JuJnnxlgRz z!qu!r&W{EbEjOpRP)$$hZ>K%*59-KQbJM)p>&B+m zcOU!wwkYs|^$~Bwnb%MyT69uejQ6TvS#yEqTl>cwgU868EZgGb?e3i?FGM$O>C!9Y z+e7EpCLT&0$ING3*by;htqr7mF-~iNmq_4Q%wI@5Q2|xgcP<3&e^E?b|L!1$=ew24 zBb?f^a9A4@7NZ_T@b7uQ>9*r0LfW*qBjzsWWMTKa!U|VG6P8ecE>8N~1;$m?7*YbK zF1&?U8UEvM4iw(FLU4?8o&n)%+b8h)jcbjbcpn1!Rn3|$ z8QZq{s~!SeT|KIt{;C!ZYIWO}7|OGfC13;u7o3*`D&Ad?0y+GSdOl$PgOoh{yQOaY zI~ybQ)63CbkmtIMV}8F7iPg()zBG3-Pgz2+>U?lKcx0J>R24O@4Mqt?eppVJ_w?$@ zU!kHfFz>P*c9q#zcKD8_pAg?hklpX9;exAXDnft94n>g&G(u--~;S*>= zu#ppMY(r9hxckX(RXI|!dVy=(nwn0-c6VIr8IQ>~4H{ZUu$^F{14iC6zjeWE`{RIa zR~ZDX9D}9x9!psWH-W@MMm?=>mG0)O9DNlGlc6N+5L^y%gh$^;^NkufC}I?}r-|Fz zudr8@_)sk#)j8Lo(Cs+tXPu??VaNglUfLsiXyF3|*-nFrvsK1pl|iy9<{i2&b!mof zA{_;c?FCG=1qG9M;1`Ps)gH;$)!06sL(GU|Jr)STwj}A#G_WMOp;C>#P92dCY&j%s zLEWRjTUF%J%R?x1)xfm6;9q(OGP3+#!1rkL3or&l2K!kN{##d;wwvZMxi~9AH?t>G z(hnCi`R(1=u-~f77~(Os3%>vmI0C)CHNf2mPEhu46gc65UbnYZRc0ie?OdejDVKk% zmU_HY;Pc%{9wJ~S!bKN~HN=vb{6~)jF5mUy(J&Tc8h|667fgTlfFjvck=o( zR;H3~b>U*BC}3wMahWCZE7@|oPrX6^%a?5AUu-=Fc%c*Bp1TXhEQ zFWKPI!MnS>E?f;3-d=CpT{oyJrYNMv3Kp)|Ql*!->Y zVZ3vxm(!)rusO;pef{BD5aZcfFz54p0%+H!GNRHnG1$AJ5L>@b@G5Sk{ zSw?$@_kej*t=#bMk}<1aAXaHEn}u^izF27NsPvr293j64ZUA`#?;EtA#uuF!B6_kd zp(MD(NX>TmZJ!_5Kbu_QMy|lm{85SAv%h?NT-!N91@wwY1hmWZl5qog{~E?k|NQ=m zhAW$D*C+7x0uGm*?!g6u4qQdHGTb#w0rFzHbddgLfYn%%WFPO4%(ubLvqZH zT2DNjaTr#}ZlZpqRpZ5uki8%(EFucH(N7+9{{n9tQV&Nlb7a@DaO!TKHxg)_`?W4E zYXv2^;PBY!6Gu8aXjF5-VR-$R@2J^)0Q&hp&in5E(dgyr=k&Re7UiIEJpQsbzcya) zdVg#3LY#~GJ=N9eWv#302agh#n`In9ZN=|{!OFZiMnvEa;S|uh4cnQina*SC$u^yx zt_@oesD$6{5Zg;VzfZcfoBVH9K8c}-&-H`fdBYuOmh$+M`Sp+pU}`&i`?>jgx|{p) zy*<9%zgMI)Rl7tB(~V*`Gto(aW8J-_Uqb@YUDv^OI@x_h*grn>n(7gul<-kJMRuB5oS zxtaU5|EQs!c8A&MDoMS)JUdiY8z`M4B1#@o8c6;A!Q|r~0e!S&nilypv9sSM3Txw5 zc*c+O`4faRNJM>TN!_2XtK;=yU%8z&ulNX*)O0_rg{`|@=2@P%S|E;FYVAd_xsNnR z&S%tX)u^L~dle{}uG7oa=FZ)%WHH|0oK@$2-mS=m^E_T=RMcpHk5bK&ebG9r&U%Pi z%8^|+-tzzAF;wtAj-F@cF-=N3?ECBP&g0>Nai<~h8@g`s(!Dqr zvOvLi(gSJt^`b9V|8=CW*l02--V!{-OmIpX*J9OmAs7vM{?|CY{H4e}g+@{(MX=6OLHv_XCr z<&ZrPisNXsm7NU!2Pu>eh9l5W5wCp@!;ey+=4bpl z1TxKn2lWV@wX|QR5OVnJwD4%lyzScAhY{sG2p(V;R*6o5!FHU3@8bX9D#Y~M@6tH~ z`c78E`84nUOtKATPrxHKG48Pwo;D=jCI9Z|X>B1t4b@}Q=+KhzJ1T?4QTYZX?v};9 z{;&Bb7!RDSD8Nc~SRZyp6WY*qZ!`f|RU}vsoo51NzTftbY-7F=sS_W=GscsC)AG^C zy2R*T0%t<=&do|)=%sp9yX{>IVi|^Q_yMmw`elE8d5yRALHgCXwY9Uo_K+{3CitD#zh{$ zW)q9~`DYgTgI(#`3nnij!8Rh#E)qyPBR)|Co_s#px1@w?V@bwclD>ii ztlHK}{#HqN5q_@8B6SPC1?)o0@RY~JaSP^-2J`j*c)>IdIHU@xGAKDFAN8Fc6W3AXnU$d)%}g`1#=om@NHQ`(N1d3sG{q^SAj5q$ zHVc%nK%PpUs~3K_`>I#QOSsA}O`z5{1?dkK`jl~NB6Z}L_)fe=GuMvts`F4J`Nc5K zXf_0Oms7}5Q^Wv!D?t%tD4+#$WW&Q2nr%CkV?t<%y$hq1qLSPoo7yhxvi#az z`l!tAnA}F9<=QM2A3hVdxTd1ES3gB!i4tI(y?^7VghrMNEDG!_A4eC-8d0E?D?Ze4 za2{|=mBx{#HHPP?;-#~&Li6xZvR_0}nMj{^jmpX*+A`p&*L8oW&!*ZWdkn;bnjc&Ps$PS+yojOG#Xp-BHN-DB|JNn4unPUEfFz9*Cr0?ulcQMSnYp-;(O1{;+n(oK^vRZavqkv5 z+GyiPgr)FzL6RdV`MF+&NSQ71aH!GV+*VkR2njfN{w^%=iF5e1Jfl~tuXUn@&!ZgW zCc`WY*IxWDxBoia?)FuIR>yt)_EM!-OQR#-7_!v~HXc*Wp|$M>?r0fA?`GIdjVhMC z+0^G!j!K&f{@K~)8$v94mK}$`k6Rm-rd*~d+23UQth@454CUDuPZBa-WKJTw|6DR# z>}DP<^1@6F1Wz3&YN3%=O2m)Np1~%fxQ7<4s#pCb6?3H8$mCJNtZ7G^&Zz?RxV4zh zvAjAo+LW3~bTl3mu94w@3}zwo9Wao>i65lfMM6dW1)GN-zt8eW&2q5GLY7#{i#Qus ziJe$&aU#;g6o)iEPl9wHAULjY16otW4~KIm6=!yXat2#hcsK+OtgY=9u;MRl3CWDp zG~gFbm0}}G=qY#!mzbN}ea$8z6la7hc{%k38nbgmB(70X)imZPvGRhUK+3p5)*Xj#5z%nO_u@BtF&(9cfU;nqwo{pS z2@eKm!abO)H-yhr-_ZW`d#q>hf0}4^X{l4j;M7LIkc2AP*;8kKRdMu_py7Gx2M>fj z0Sp1qsp&4-&7@zEC9G^I3B__AQ_~(BhnjQIqevA{)*}ajxY!!=KTL#;EB}=K8T`AW zk|8pm4Simi4KVsWYL9WMl*yfWw=3|E#5QsJfz;`Mdg5UlNwDsUQgA#LH*p((%*%Gd zB5)#9=(Gr&XB}?}k!v)s?YGeHU0H>&9PEQF4hskQ$?1^3{~l}6B$HvE+R)g$(oCd- z7R*%EcGRx<-bDsC9eux*{W^@j6=D3mHUK}KmRZVti%H2*xcb}ZP-_~YrI1s*URbCK z8U+oqJrbnBdSNwp-(}1dd~&8jZX#SK@doOp5dI=ivaA3Z?oepIsAP(ysiITE5R3Sp z3L;j%#-ERiE6m|U0&8a`P9b*h{AheM>0t40)~CjFRzNR1NAKO+`!ho=M-gqkMdY0g zv;a2T3>kTam;4NXQYQyqrF4Rkh@E`pb2-&Cyv!6QeuR+T-ZaAf`)oKv0vQ(4Nj^wR-5Xz z;whWJ|29|h!(I+GX0!|);DqrEw=K(VgwpDZY;5cNre8A9Bf_x#~TJc{A63OcYE}=dS!n?)&8p zNRKHOCUCfv#2n@TSkt7f#kS(nb2zKH9{-3U+C^3(IqCXJ<5qy%biq3#J;^c;A&kJ- z$CLBX)Gk6~8&*EGwT_i^5skZ|BUh8;;fooeYf2)B_-tnzFMoUV7ax(PcCd`OjL)=LATq-!lTPf{^8;S? z%NP`;X18#;FXe-?yq~}9khPURRu&}P`DoGLR?eVD%tTI0oN)2s$e>`hk%Oefdmm%* zUd2i(i859%&Pn_c=jz}rNdrMYvWEp0sacktJFeaAS%QD3cDSo8(9hV9S3-z!T>ElI z)IvpK8P!py0=mcDBF=gQ!oTE5%z`jN3b2_|s`#C4-+P_@R_+iKbw)mo#A+m;8# zrgIcGg}I=mZEI!%j<3C1WLXBa%}dOnLxDoG&GCU?l^)z?&bA$!*iUl z?Ftx32H+%VLl$ZpV0*G$UsNc0uRvkZ2abe^=N@*2AM$rAt_qLDB9&rr~D}NiU5D}6D$t>`i_9?#P3WiRQvm^0- z#1<})=@nohh$%0J9l4VbWpG`I9CdVQC~zU3(kasYbNf$MUWC-<&&eZ|ecuf&r5pUh zcz$c?QI40j$7RPzd#gxMSP|05&GRmB10)}G{3p(nIiE}oavr3$Ra7$zGq+1LWR>`+ z(!P6?f%I>^d+A>tj~78zwNZ5+Hi!L!n=_51RAI|VEar=seQ9(&LIg$JgVX7;u(20& zq04W__*~qa5Z;sy6mAEmOsR%s189?|yP=*5k@2_p>%jAz0hIgLXmineTzFg47{}V% z>aicux-!L4Ak{4@lJl;U-17{XX!F@)U3T8X*<HXl!WdU zsbgx4uNOrXAD=VA^Ur$)3)A393K3fmKLD4VSEW~vXO6-cuN?5))SBUMMk1VH-vh$d zoPd(=yKS^$Gr@}VjS$sAz_v}#U$5svxF^U=wuw^91<53FmHA=xg}+>q`KtvcF7`TA z)oa^c@%<~5@1W849^Z*>3k{!}V_WnvNwF!(E&?INzUK{1gf$E?8m1Xqa=45b;KZQE zO=2!-McRQq+U|)0W`>L`oxLvAGJ>ny`!x`Tmx3;FQ%@xuWnEen8)a(~%sM{d9XGK= z*4D5~SlWk%Jc=|=1Sy&X(trH}+;10^k`%w6(+$TPe^^yO)9PE*(BdSfsA+3^%nI>z zrigGAAue=EPgs?9!eu8NCpRG`mL##|uqh1YvR6p%bGWG8N6`jn@vB2Do*a3@_0yWF zUJ9%_Mr=TzCg7#DFut1}#|*=fg4L>@%`G6_vmq={Zpp z7;?9Lx-Ux}$O==0>qL4K@6wZ^v1}~yi7Y{1j!-TWAXN@GXds`nH)Hc7w3CM9dt$O9&1|Yyu@;&j>-}kNL z4G2$vdda}7r}23uH(B=|Gg|ukggxGQd4}hDj`&7qItV|Jmq4I(u$_H2cG? zztI)xsUs}ZoN<4s^l_(Nl^SAQiHl>>v}$PD3%*2@s^4ppPrzf>z>HV@=Q^{ z%jOvc0IxU-EwAo*2;AUzq4wNcr$Bi zGgRdEvakYj!<3|-I@w~~9i5=KcfIJ>*2DVC0-5iE zSWbH0gHqz-8fz0_>eD^i&I-N6*^C9pVIQb>mY8+4I4v%rZ@TZdX7jisp_@G&RCl|MIda>#63G zFTS#EGkD&dQTVt=tI5j&!-d@gF45;-&tW1od3cn*h*GBm5_WP9>NUwmV1K~*bt zBCZ%tN)D%CT}TCH`2j)EZ%P-Qx3>jUU&@CL&SAlJEFQo|S*^T=;CukFBm8u>U8yP| z)0_+YFq*!dym~SAE4ByUu>A zKib*9$qjsk2qyzmmR8EjFB82XRHXW}n2{X2qnehp91LYy-z|0?$uA=X;h@C)FQPW^ zMy}4HqaMJA$12e9MKjy1^@tYIbmqxnE}^c|QE> zka@SP%?A+>?{E8J?GzeAEADW6UuZezL3(=v*En7goGei5N3)GNTowP{U{*u4ey4?a zleAmLfBG!=TqLeY6=ICy=2MXBiZ5Dj|6rrL>!fysQ!6zFC7A&?w9_cSWqE>PoDJZ+ z2?QEo^0Ci-K5h+}y8ViH%s@Jj?E2AZWZ5^YPc|091Y<>Lf%a6t2=fD3^jTQG=}i0K z>(lx9j{n7L%q0zY-N#=B>Pk5*T*bmQi{zmgCmV-N){cYJdQSVxy02L6x1!W~KI@dT zshkwn*gQRVjV&GX7&*y4e{)9--O;0fD?#RVgxS0%`0qviv-FQ$1@!;+KmB5I=;*JM zJ8CC+JWUjgU5-Sm>&XP-TWr1H87keA!4PE5t4ZW~2TneLA+j0ltg|FbZ@NS2rz7bj zb=&I+Bywn`|7o@mn7`)ly>|Fdij4Ccj?vxPzj>DfvYHj2^IfS_n*B{hld;C8OTW(S zBg_!Qv(EIg@g(O;dlgIxMP=KXLRc%E8ltuA!v77JEhY)$jotY~PQVd-M`r)8BpWXU z|D*o@NSI;KJyxg%iKj^ah&}qL`!hO)2=T3K&91-2sLAlBLoH;e4`()s5w>3=6*2{H;e;eJiosd;a_9t6e)9;)#3+2P(VU!C!oRu9r4=^GncuEy!C--6` ztOzFL8BFnDV=-Q2>dEGCPZLM!I zu2}tv4iU>4jnNog(S&0UmBbf0DI@5+Ypg~Q$_59!Xn&YEAaaOcLG7cyF~!osBaZGX zcYpi)g|jh@J@r~!GNCvS$wwGkKI1?VmH&lMZjrO6FjOH8 z?q@>-`Qh-v9YY9*96drRzU!a$!BEZ`MXu!K`97zu{q{DKratCvpkVzLYnKD0au9jK zE5b$9rFwoSt#fQd?_4HNC>dbNJiXhBOf~WU{#o7LOtF^wehtSbWf9F+sH%K)Ix100 zSTNRjQy#;2_6@&)6ZgDPR!m9-oduwW3`YRpJn?m|)aaw#>q2JnBlCEQa_tEtPwZy; z1R(wbIB#`WGcuT&ri9bLJvKh7)zC>B)quCQ*`EV=Wj%Jn5kx~KbCsswZIeW0rV0?c zyzO9On?m6kek%Vr@N#N1*#wH&kt?%sU-icTe?;c4^f@n*-lgmZaJHpjG`4?&rd01} z@(+UwLZTdBnBu5%oZJC0NolExn}&-wgQKt?#F6KdZno}2kywa+DbGE$phOzW>B@3$ zz>?#C(sgsIj;Ze%yY>r9VD~K7WrAVt;}yRU-v6|#tR-GWY5dlj9{$c~P5Vju+azf{ zOdl|BhR@2Y17p=k9C#Uz7wQ#gIQeujNG$QffU zRJ8V6Pa)3SAAulpxTUK&{yTfc^aoo-O2c%h^$U6@q0`4Lv9fK3%{Lo{!wymLl1=5j z4(}s1Y8c_K8S;70Xz8m* zY3d9)*~eQfM!YI4!K+~*s-Fm+Gw|vUX+Mi^*mHe>dvZy$c))Ies$_o7^c=h3pWD<# z-zOmAqFP^AetJEN1=}ICOdd4LHk(1NQZAiTO!;g8*A=A{A0M54g)eQAt7&zMY-1W$ zpv*=$>crP^tp;xsS};T@MRFQ<&*)gtYJd3np_h!=Ifm(QhA~f>7k(!bI(|yK9j!A~n=7 zn?G;(o^p+X&g~qH1Hj#| zctJZsMaW=_ zC9gV1pjkj2nckd@Mp7GL2q`5L88SEv)JvhFbKaqv3mk25BRmGm~-Q=LDWrMO)4k zy(-XBl3`h8C@64D9k~|E1~bG9+Un;t@QILCIRQ^XXdMMw zsk@q7eIK>(QCust5cHhd?#PUO?^krRgAhmdMUKHVcCJ_g=R1q#EfY5fb`XDakQwzF z@!j*_J`LC$+N9KgzH?0|R^!FehNMV@Q`k<;_C5{xz%S2VRd^S$(@8`E-8^gj+o=}s zEaENaI-C%hBjNoHUE?l9zO1nwdI-HlN`Ly9X(iG)54UV@bWF}y0;<){n@|mBEo|eL zonl6XSROYQuCG#{4Ba3#?3?LC5P#Ep4%GcEmU*32&h|5Nya5)r!|4^IvGdi49HI>A z(dN8}3^*3_Bd-v}X{qW!5yDb1McksRMS&6OCCE=uy?m zyfn{SO9puCLp(vZ+N#eN^DxYBSeb*n0$FV-E$jMo=r>Q!A@Ovxg-kJTeNRRQiw0$! z##M|%8;I#;-tOysD`8vidE}Os2Q9=|cnsokX>ht^a z@9VC~EUOzjb8Xqz`5HBMd+H21aG;CyR4~ESY5#ko>xxg-Tn;!UZkKBVUS1hi*?;C4J50^(?cV~BDv(rc`X*-z~8U{>RoV|Xl@PtrXxWrLi=gTvL zg?k2HcM5F;#?J$lZ7)Y~q~=63dK)4$Xm*D1-LqGBYfD3EKsq0h3ONN@4W{fbIWDSB ze%8K;mgfZ-)s3R0{joSHCi%grD@f>#j;;s8T-d63udzVabMpat1TP>G;gD*N?v_N* zdfPBMoWJS_j%xLhUR2g19_K03465*kr8TUIdM5V}M1JgIvtIa6X%<8oh2fWP8c!vq zBiDsv&v$~PF{|SBIa3!JM$EV!arYSN1C=b_3qvZ!>(+Tr@YrcJWb58=s}lB4wd z4p0w1za)Jm%H!WzWDNRWm>pgJY-Vk1>~wfJ-;<_>qJi&!l3Mna@P->>9Yw|t7bKnx zGlk#4|6^3Cy#(sH2p9cDIG1Z^p;El(_5-+wb+QWlbm)EgJ7I|>riua>+GXa3tz0>d8_Zb{9Dm8<0@;Lv) z@{EnB2t4z}2MN}6 zEB*Z1y`0~h2W-ek$4GldWSa5sD!d3Tu_JM5{&4(WWi}ofenM&t6}5+B^#3sTjxn}% zT@+^7wr$(CZQHi3TefZ6HgEZsZQJg8zwV@y?xcVB?|t^!XJsY(%r)jS=C~DY#@8P0 zxY*1H_Q5SUE{54WoS!=?FUwhVLt4b_D9V2u_%YKxVF7>eDC2{I=-MP10b3Pz)EMlU zFbz$ph)VUtnxsAhIcIiJMWak0oDr=|1)YqWuU`V=OZA9^azTs7OX+J%`84t>lN59Cv8vl~|`w>#V1(`u$RCx@17I;T%<@05pE1iRaL|Cb{A2R~cdbo=XY;CF6z>x7tb61({AV#EImEpRDeBG5 zqvKMS1`>TtdbzM~sr8*w`|$R@e~&fOJI{mL-rmg_F0Wr#Yqxv;c-(rSH6;qOjv#X}KaLFvJKIpXrFr_tmR90MG z2U+M$G@1{Ru=b0hiu9VMw6blt3uzYEo1Wug(ICo@hrBW;c~E4yZ@{f2D3(|xa;;-E zgItf6n!N+92qrQE|Ay*}qQNt;rgAyyfxR7Hb7-5~OyJ`$tQq`hCM^xMFzbBSTYu-4 zh`w(QIi07*Q7-jRwyntG|DBNtkK!sa0ZyhcPk7W_8}h)MA*ezgwVj$Ino>;wHGyaP z21KCs2PKwW-A3Am%WBL!+&kx4=~5g?^t_qQZl>DgM>erps3|eo&aXntw?M2s`}uE5 z3kvM#QRSNRJvbfT_NL#biy!tpNHQCs$!cj+I8Bv#Iv^u%&ZCD@+@cfW*?_+y7ZIrF zwSb6J!1K*TU3eeg`}2p?4&T-fdXYW7Vso5DtOh!sP$p5CRo-G|8r9m^>?p<#v#g04 zqqh`#F`vi`0Q()Vn?$wytWuuhRqwLRAb2ew{B&u1(Xyg>uL^mOA`d}^oit4OE-U`{$2O9MsdxLGS$FF&xVfVkOcGu4kr_PDL@X{@XZI2TEI_oh&4JAQ3zumz>Xn&56?Y7?r_!^zd3s{6rn4(=Jj74>(h_R_Yt)Q9TSOB!a_Zm`%=CN>8ZgP^-|hGkD@%^$Sf7&TTj41 zj-m$Af;RpsmC>OZuoARET7vVbA%eBsJmM+tdTCRWx0923M+6RYO@$q;+}Sb`7d=+< z8f7>9N?xCz;i)juKLA@%qbRC+9`XaJ)G4PH=Wp*IFS&~!xUpwdYd_Q+x{s}t>JgG1 z+j9#Dh(_M~v8e=cn)CgQc5R@T2$Sa4GcO25m=3}o2X~AwSmoxU@Ld*kkW-3&hEg2m zFU4^8+wvD07Jx&H+tbAzzONo^CRO4oARADObA>wo_qpf=dAaO_nBREBnwE{Oxxxu9 z+2ywj|4^A{-Br*_3STWRAi)|7F>gl3-ei@a4cpWg#>K`=>`Th3TORka%7bz&ay@oT4>vVNB z^Qu@(t-C{Ak4vuZQ*qpFapg_d)iWVxD_wFw$qTLtf6|whe?uo&m6*ei)611u-2g$R z+RcP&C%(`7`%qf>%r!d*9Kf-uP(U_E5zigC^T)n5wH4?qo`qs5u1v%GxlLKhZ=1B6 zv(yf%((*DBgbJ~}I}Js*ixRaUie1jQPR8E@t&z)iKiV~141N@VqtqWByi}VaORBxR zEd7QX&RcLUQmqpRrd&(>h9(TfY|HIQK+zl5okQ)|`sSa4Qjj%{8?Wr}YRzxP5OyHoc<;AHSx&ipN&57;J)P`_et5yb-K zp`3LPbo3M(PE^#&@;TQ6p`3)RG#sT!sF;NnO>=kHoFtlgSX|arS29Xl|Bz_i|mPcjqRS&$QPC)KXc|Z zL~+G2QCzFbjXks261j4fI`c|%ec4eK7U@e?X$82J9;U$;2Z8OphZ{}tNt4~~ee9^R z%Kk-R-NCul^7+5MQnEaM`sl81y4O2DJWU-vtz=DIcM7^tv z@E84zvfakp=T}Eok2T|bTOn-(7p?i!$9y(WnhD`ekK>d;8e~-147yd^6#9LHBeZ^D zN$1^K{AZ1qWuD#=%A;^PCt z)sn=NW;eE&;TuNiIL_G!m8YRi2+tiTW}kCVNV9BP3I_M4uE7Dc}ldp&cys8O7f= z^^I?X0VyKynXKoU4dyK^a1pJFjI$;O^?D6+jewmh49)1sFf}(_2L!G$B`C+mwfsdK z78wv`l2%k;Sl+|IIfh@kp+WEkLBOpsoD z-#JNdt!u>~!i!@g1KZ1_0x2vSD88=);$ah#(QdBoB4nVt0wfg$mKduYd%f+ZJ!zi* zXkf`nyWpk{ByTvXXN43*g!f5}r41ZX!K_!^LcxMEs5iGBI0NIg zi$w$7RTm(c`N(K&Kez}J6Z5Z< z!-pMk@q)u6FycGR_V>Q^_f|R(YD0h4(&>zQHuizCc*edG6zY{l!Xx=h9Xie80hv^; zV#KsuiD{A`BMRwO5#R&m7Ahp%8X{6TX|%6Y77&3e=MzzlyM20WDa0Ltp6Pk77L8%# zNlQG%0Eofaf<||17?1=j zt(tu+Lcvauht2(|mVqWN%XlND{=-L4R~+}~eTgBv*H)a*U`j=s?Bl*5Aj!uHosH6$ z6DE838+d@gg|)yo7ZP;7cq3sQ8`hh@;BaWN6ef8e8Tw+PT2^Yk)UW7=xBJxAg@5l_uqjLRaEIx&1o}{07RgqUp9d3_B>$}Q*U*-yOP%)s-Y(Pt8}mxrL~1{HPv zl78e5Q_a*ep%o(P(KI!JX+egCCl5Z$0W-y@e12MU;;hWLBx+xRbzpY!N+6KzJ`NO% z$csnyK%z&~$JuKHkI0jG}8Gqm&7dHq_^#OrEuO;sQ0Jc_%%N zrKIKMoalxLH9MekBQ$5-kug@B?4pNEkDz}_x47E<9y91^1zl zEQ?}!C53-!t%Y6_ftuS7Eygc5E}x#JD^^khly|FjBbK^kY%c_PDq4oK-4|7s3#eau zCy^I=@KRVnzn7R~ro5_tUQ|yX>U(#sdf2QU;k&gEjG%Qp&v;ARyqlW4;9%DA?%NN* z2pU*}29%{7US4Cspvk|i6(!-0i{wOZ-ph>z1Brm1yNURiC$|d@B8s_A47r?@iya>r7rax$yVat-YUyfyK6HDfwEqMON9TcM$J|{tcYuCmG?rvFoduu<1)Ew; zv=YysjVmr}7)>GjwH*!bV;qtP#Yn#+Frvf#Q(}_HI|7l1D~Ci<^tK#jK)9RF47S^F z!MfcINOnHqD@^Qg7YF4>l(S_#(?opa9D)oq6uE6E0_9DvOpXt;oN%%<^xtt zkltz-Q#2FQ&_Q4Z?eF8GaPW_)pnS%ZRM%_Jao(vycfH|@;3iryR`8yN%YOs@h~vxf zTyyE3eBAVf=Lz>_z#sx18Vhh?5>9yq0TKyc{u0^Iump6EjHfg#pAmdL&c6NdtNuYz z%n*0QgoN9YRZH8(KRGyUaT~Nn z_?JNKe5|#7qIvctX1b^cbBalQQtYiXVWZ)tS` z{{1#XDT13u? ze#(V_zs7WT~GsU{{x-uirE&bb_q#G_9(%(7O?fotbqtG;5;5J42UE>%+X|40C3*X4apFbL0;>sr|N;%cPPqu^!^M11k_ z1X^%iF^3LoPvlx$Tx-)=cba|J8q@RWZcx$`^4Y!H03Y%=)k@DrarN>Cy#D>TTNK%c_UR9r@wzg)tnO~XB~G5TQ?Q=5g#*2Np;_pCWF#vMafP*LR~T(7ue zgOULzzZnB3?CHMNnaWR#d;{;=?G4A*#RubI;nwxy8vu)zWr~($S_}kqtB<3}rPCP= zs{;21&UK%C0h1wUBdLhVZc2lRBy@+7B3o@!;+OJRaawC~-O7ORDH!c$> zbx$}(4B>Ht)H5a;>F9AfFi(V0Z_AvUR97<3Olh{7sCUaIq19#Lac%bkJ?fUADC5xT z`80=B|9n(5#{&GqXt$b4eulIgNB;xPwZ>_6HvRz?)D+~7Nsq?FWXcFk*? zp>CS-7Dd?MjP|jLc#Gam#WzqAZD3{D@?N@_DAu1hPa*TUqmbXBWAP|R4TY3@6{?2v z4w5xwIlBb+!uW?7(VVqS;+_efotfAlO>Zv5!g1fDE0>w~H!5yDN-?WnGd< zhZTdG5zQ!FH|`T$!v;H0#mqP>5L_!>v*P5 z@c^A4i{HZw;bO9WH0A5A-fa?V}Qxg-=36MxhXR8SC5*geX@uTn!L6l54^Bv&DtEh8$>3faRJi%} zy`3y}7cpgE?czMQ@cUQ4yLhY$Z0^aQ+7ldrDsh4ze<*RzoYEka0Ei4X$pwtk0}LPBC}4l|Ijr^|M}haeG@ zdIRW}L$J%bW2ejc_D{S|dzrDxgV<&`% zFSTzfH?4Ov?A-iNb+46JNe+cgFfyvb*GjTmJ+ygSLS!~-jORAq-`??0T>Z+m%+%y| z`iB-U%T+tz`kl7dU#qbW;di^?bc_j58EQ%NG3krB;EyZ6+H9a@4FQAGh718`AP$z6 z=JV&Nm_)@M4fdoUx`#+op+0nnC}qf~_PN5!dI(;ab64EVeCM55+WxpK=^^I!^ zeex}7xS-7tH5bDpmZU;;vUmFFMgAq2?4sYS%C@4yNiUKE{FA{2NUOi4Bgr8X5O)HK z6o{K=1hu0GMLfTw^zWaWM*8m{0v9%Y%gPsxVlT~DE0wy7h7v~70c`pVOAEc{g)#>* z;y}Ud0);mJtG$F7Z669f^r~=i3R>g+d=xChfANE2x$xMCd9?W0XB0LfYVCt!C~S?8 ztK;-g*ovzWgJNi5F~tCdK-YFpD%wP&^(h|MZA~S zwY?-?(7bUdedSjW6u-jTBpAOSJg_~&t2i9a=@w6m$#o;j4pFTV*(PJks(vw{sc0av zGP{C+e_gBzMyJbe*!i&T1+3Fqo5$>4DVY8)qJPV^p{IE}35ynM2D*O2Q0fLR{q;#d z0f!N>&ziijk^s5TUAf}-k8>b_Got|*Kf~k6_8jH^FL^i|?cWm_cxbwJ8^l=#-bKL^ zlp~KMNK3gJsEW0t9x6A~Dd-PAKtX5FxSLw}dh~DA^=$Rct4P1n_YP zOJ%xY9v{rJ&`L2=qUZoVHmJXk|7WZ3%5^p7df+J*HgS?edtN(3KL|p|X4)AS>}@@~ z=b|L24Grk(c&N4xU!!AwJgnv()SANhs2BmZAx@AUhErf;Tu%Y_1TBiWbZgwxKTd zgt?k%Mp*N}^xpQ`l=>eiJuPvMvyW%PvOIR`_wQ-Jz8d^G%-qNMdEG2f`TO~;oQhxz z3zSi0re`#0vK&t+nMpolgM}<}@FY0YbP{TvKuIW>bqI!v9}x$HyajqdAtf~N + z1iffY#b#$RU_2u^Q-#MthlE#z6f3ymF|HY z-~$6)6#JP*2Z;c5Tl(U71Wcp_Pe1);dlV!hY8^sCC`e6n5)nWgs9lK@B>-`>)_jyH z69?iPX+Ugxm{o{+r-a1G0)ZHkI3U(#^nb5BM8N8SSSbRqgd~2cSUq1(!q6Yzx39aq zr{x@cM}xvAOzu|?;X?usmaQ#c%D z1GXG|3*)icDD{2HpiSkIDOKfPRB(|-GN{H>#xh+lU+&gayMK&SRQr?Gtd%b-jns#h zd4apA;*}`fR0reNU$Nke3+kvF8{Z@-9c9!M|6=MK^tXr+8k3Y&C&a;W+Ug>dXVzrt zvDChg<7swG-{moL+E2NDa}$dsx1#~icENGtUDApMhl?YPGhKJ zk$G+43&lWd8f!DYiFF*Kw`h)?ELTwc!@M7IYV$G|yN~hFE&p$vrE!BJrHv{6s zE=Aq^t=t!ZgEkP^M6OIsfbyJb9+88}Y$)=SjLsp`A!^HL8lGxndBU~+q-*7#|3*&h zKsS|^w84%*%brscJS9D+<`&bHo?%Xxli^{HgpG0ID|=n>AH$Brh*dhwCo{@j2$qp# zC-7qa?i|p2`o}>TNiMK~86oD_3_1ZuYH2&g&%idy^#4JbN%ZT&Fn{}oIeS^%TNmB= zd3|nlP6*bZV_>3D?}N{;g&hzFxoO-q=eIJqb#&4cAuEMgg?VjMYahXRjXG5_3hi0d zc|6*OA)2bV;pWHto#7LC!R5v!yhb(I($H=H8xvKepUyrJ9>X?6$kLaiN2|XLNxXA!$AE>Pod+5`nh9ImAe3Ho&-ALQ-|`zgiN-(5Z{!j z*iF3lWH#PqUazkljxno7s0ywOmWoHb3|z@f{%%OczLQMROg=(rT0vDk&T#%;?z6(4DmxmD-^qZGl*7zB zxUgX$5zzA25byHjx4=OKaW~tp*1|&FyTwababw=^%J?537qYXk$I26 zoh>i?>CXT_`gN}htJYz*7BM`(KX~rExuqzzv}L9*M^|r0FPFXUx7V@x>EQcT_54PnzH8j09)^is*zSJNtMRj^&!yiq`Hy4 zsj}bV;q*cjBO_;mz^W%j72&Bh$%dv-c<#PzaX|jgbyfsZ!2XZf%8Ek8_KzzM;ZLd= z@_hSO`lsl+^WZp(Wf~cD$#sm?k}X2ly_~pvBcB;!CDYn@iNc z9fqs=$=4mHcr`v*xO)I|M>SqF(u&yajYLA}cP5HNtlM)09hd!XN9~C7_1ACD2l*9T zI$BKvmNQmbfu&Fxck3&`G9fP3MBY4hNvjDG-^V2!2jmxDu&h^hcp^WkA{qIoOOB1u zxfKvw%9`)&u?j$oKFrC*EuVBfsK5Jf&<`#;ioGK+$Z$v04-A!JV-hq(#&*|Wus`|n z&md>!MtVo2Jfob6N#n6(bY?gXjg47gW3fzg2pwh5J(Td9t2O&?uoG;r&$rLg$K%`n zeORJ@Y=YJO<{a%QQ9Py|;u#&o$cSol7)17>YT|8cSjbk2GonEh`I=9iL{|2YKSg>vcOp zIF9~)R;{zcYQPZix4-v|tzHbxOo#tJfSQL^Kycg4r*@JX!!E#e#WKL=$LH;hB3(qQ zi32e(A`U>3-w>|nnzx79=_u&evohojK1hi(__gAlvGb$T>Fd=zJLQ6@d+E05W3ZpKiCI3$y zVO?Ub4;4{r!eI`vRD)=vn5d1vff6h2^kF8=FhMt}eMibLVU=8^A)=X?Wk7@%4Avt- zbJ78Rkue5W0N8U+C6{{9g>DjgCruh`J&y6k25IR|_bB zU1dxJI%Q0(FggVzK|uvY_!>XF)yb<@`MuHT$=moCC5UvauhCP-mH7hjt`gW9rzsdJb7Z)55&PyhHs4K-+Egt`D*r$IG`qL1YV9^*Q zUXNq_9f32reyU?^!ngxU`!6F`-{%k56AZ|EN z#%UlmZ{jfENul)jkf$(=5s8?T)>tUK80ba8y-AyhD@3yo&CE1WZ9G0HVG=O-EpY?9 ztRdCxLh8phbr^!3pENab7&_6?s^rBgD5N?M&ZFzNM@TUsdKOC5Cx>K^Ogch|`t*=m zO`tw$BguMrIt0(hBopi1TGb?#IUXiILXC^wNF=<-Q13i+^wFrtWJXogUB+_i^q?iE zN+;MFqF6L_N;L`qrOldBsY{;sOr&~ste&XNA~>n|N|Zyzf&rnW>MDg7A|+$VMYU{b z6=ch0%BjSqNVGR%8RY*7?3OPVw~M@5(09%Rl%d4=noz&lW$Dy$@#*pPcEZy$L;_Y3 z8%ZTsmPdD}B5Bej#%u|@dc&1q6UeI=BH_8FEfrBM7ej_pt=J05v0=gmjP^Gyf%WNc zE!ZQ&>V|H1VhqKKniSiexiXDbvZ>ss2IU5g>RM?wamG4N8IyN`ECfT+QU#@QZbsUM zPULb-x$fc_i6nM~u5xw3)RUc68PE$*yi#`pz-OHoZ!&wytM*z2dx@7EM5*XGHKOSR zLCeHCEsnXgbsI_YrJP(|N#uE)jNKK6%vuR$<2kBI5Xs1N9pYoQajOz;zRtWVOTaz- zbotLB#abb8=@N0zwb(H1fqHg-@2Y4jmep;f6!%e9mONJx)zMy55pQh^Dbut?T+_dW zEA4tosb#O`{KZ43iAJ={)DvwoH`c8+AARPYmXRiE^GZhjublgL?y27i$=%;yrYrs$ zdOv&JPg4pRkk&meE%ZJBA~7vG(8qW|f20itF)r!-VZx4t8FZY!>?be$n=ao)NryK9 zTvE4zin;+gu{&sVlqD=J+=`6ZpRYy>w&vE|gt?osKOS%D@61-`xQ4`Pk5NVljBt`5 zgF$}AvxupEH#r|2OLSu%wMw9Bk|{#6fNYmV1X)WaBYlvcI=-_)b|%;bOFyt+wj39m z;*nW=qiNPBB|uvzm?&{qNl*LHDB}|e!R+UXk`|1Ufc}#gr?o!sq1EzBzHXM8i05iZ zjS;987hHlM)O8pql-s~coln4^mCLRoz|Zaixrg|y_Q4~5q4a_ zW=jZb&_;#O!eI2Yqf+j42LfzM&0Gp2kj`RPPaDuftcV@FfFIpmnAjIE=Z6TA624Q- zWW3u-FxPIP;ch7U{py7mSKDN;v>|9>e5cO8wZ;u4AEgTe6X(=>A}-}A3zMn0rM6)i z?J{z0rOMc@7Q{>avXopTO{Xndio>z!=g6i-`Q+zfu_&6a3lXR_SlpL+5tHxoYP>{bRL?a)&j zuEy#BQ`n*g$R2r7V`Eyz7!DlmSMN#>n;02R9*&LiMl_`Cn#u;kU&Rs`94$F?Z)P}S z7HS>qy0uUaf@R1;zl+94d;bZS8|>jMmkX}+;dmyj=Wl{HnQ#ZZ`@oy`k%EIVXkgN? zDYCj0tNLxi66xp9_Rx%fiP{jPi*?<;zW;SU6Yp4`2mRgs17_>8w2xd_y#MmBv`>rn z10MfKelX>tp~)@$GvXf4M%Pd3$sbnS#qd`e^b8Re&V33Os2)iqmGRFL5IYS1CnSQbrx-r;`I1F;XjqiwWHERC^vK>>8c^o(pcLvfS48#me+Y&*)1uWgOoUkYewAd3@}kctdeQ<^wB0G ztP>aalm6&|{xWJ5Y7yZ#1m*HDA=bq3DPh)TNGZz9AlAWaTJ+`o;T!x_chqi0-0j_8 z@vS(c&>;3qtHdCdV=B=wR>L_2!2`WgDx#B^aw$PWoun#D86@fuAo^tP(L)c%WRC)Z z@Rz69OhJzdrWk>fgaZwexTjzcS?`IYS%Dge*X|sFX5~SlXe7&zPY~J;@a2*%!*3#{31}|-JNMU!U(|E)RDM+tbF`V5F3Uyt(cRJX zZ1Zfra6ltgS25wEauQ7I;9Qq7&NOAC8DP1L77!$kXo`Oc|cd47# zRKVX2U$EvyUEG)8w@JT=#^MUpw`WJzhZLWZ>GYxeCHQpk{^v3ZJ_ zWd4Mb&UU0KMgPJ{JW))Duk=L098=DPSD$zPIz4>=)G{(@^@W<}ERal=QZkg~&#tl7 z_GhER97kLy_W6d8J=S=Oks7#l2F2p^Q!vz!pBh^zdrcK2*=fq>!zVEJ1HvSn7=r7J zc>yJf10IfnJ_?_vE9s7pF&-VoLKLlpVLTEp-(XS%_*w}5LKVqy&=V||H%n?z0rS8#cmbK&V_YIk%h{DT=b0Z&YNEt+;$@DUcUqj+99Kom2{+P$QOXSXLf=QdCLUC6r9O=m0XL%%__8)Db`(!z-@H z0#Z22RUPw02?3%GQ}~A6vY05{5lB`3^}K zXutEJ;&%)AKCdM`i}(*6^+1d#ubSMB(Y(~|Km#G-Ae8O>e;4f}k>L;ny~;XfE(~z8 zLkOyh0!M=qz#TJDeOg{d<5veLXL1v%tTuoc|AylzMy4 z(`l%`(~=ksu}_(;pPv~_i)na-on4}oAZ>=k@s6?FOoB;x(LTB|>7BZ$=OGQyNoG(U zm-0i`bq07pYsR3Hlb#H*O2{)skn3L?Nsjt0PPc=H9hwWJ5J2Y5&JBZ~e>gFZ4fx2a z9Gt60wEHp2*sKPM7qvU?RI_X$0RSkV%IP7Q06;@hl7217@%k@AI&1q@k)yO%EQ$IG z5gt;R@c%2Y1C*#d4hZJ|MmyvOUT~&|UT|a(-tkoTyon%;By0q|FzJyeuzJHB4e zKG`~#GEQMIThA^Dr_M3W1Z2d?d2=@pl4TK~7GCM4AR&qe;e2{~+q=V>?DN06Ry$2p ztvZqc{gPws9PGXHeJ_6TwWvqT!&72G;MU@x2$4xtT?hMy|M|n><4WT>pl)gDoBtI} zg3*5~ETGfI@^D&WaVrg|w%cK>z2FVX&8suAykMeI64D{RDcDigSgcI2s!-c@wP13! zK(C2!h7#|i4fGK2KfAaOyY2n2{06$v(mv#qda-BR`|Ivz+h==1d8{#!QJQ*v6(pPpXLQ)$~GGGqSyW9SAq=RAPzU{=-R*8kDmH8-M+NU2me=gXMBlQ0&*skOqHW= z_x(R1m2DVoE%Vj!rHoDz(W!t5cSEB48R0@nV`84Y*%1GI4}(bO&VzFx)-hc6CT3;h zHdNVi$*DV;uiQV=N+qGbk??MJ5Oug__H~=CY8&YI5aw=jJDNO-PTV@14y=Y>rpF(@ zI!hMY^+a9aL=6J9FR>a`YoRj%A3~5B4yF076TEsa2!Qv`%Sw$dY;7={q#f&Qejew?Dkj3CQ=n$;7=Z%V@4>64 zq_vjIYtX^eU=)(wb^Eq5Wz9He|H!o>QC4HTC1F-{!SI)MNr^G@_N)vV@)E5i0_m zeIikhM5wA^s^u5gZDX+=F4Z<<=LJTC$id@cX!lJ0*;`Flx*Z9(zGwL=$U~&F8xFO3 z*;;c8W0As)9J;*-VdU)cWTa6{Ky-5I{Wle2Y|-G=FhXps5XXN7<6i#d2>vR1PzI?& z3q!e!t?tMOI-3?DX-S}?QP&tcKpKN8h-BiRBpVWW#q}H@QXE|NhdkpCXCEA;@{ZZ$ zb(%3fZ9E}Uf^ay=i;XJyYG;&HY@d1s$4s_?5Ws|ls-O>9Kkpy^(DwH*1($zd?+ZCc zM2VIe3W9Xboslv-p;nDE`1@W;!utEvMC!_JenXXsN*vWGlXzyuZ$uu_0PO%q5mg9> zRKXqh0xLVfD`zk7#EVQ`c0V^7=hp)8Q8C$STbqDQvLBqudwFr%goXi^;rCKYU~VMj=03@u)hY2xPk3WiTB zJXK5{lJ7)s@d;|ozhown_TA>yOs%$yW0wWD=!d5|h8yc%JAtzHNf8me|ts zWUT@l`IVo4W3*@tM;D&WKPEqEwtj3G_=`-88n!goBddc3H0MZa+dQnB)P+%Ce%Ocd zNFPHQMUle>cV39xNCCqB8VC78 zFgWM79B6Xyav5~RO6uTaJ2;|6I2M|}41I{DkBF| z{%lWf?IrFhTqfEFJCS1l0lPo-wF3HcA$eR^SfpWY@^Ve}Q?egFlXdC>F|23T32Q~^$uOKat8nYLO)!h|3}+PPZeQ}eo6K|1U4V- zRpulkhG*6VtYF*v%v0UKxVs!#ewy=#=0_pRXhv5u`iRv%jPHz0<$A7)^{Qj5psoV1 zjbewaR^1$iL`$DzDR5^-xS+#HojU=+&9O{6I@{SpWp(Q6 z><+G8&Ks_^eBYmUsWt7KPiaTwMGHEqYGpcy^gE*8Ysi}ms(>p<|BUXw?@>23aP54O zyhsAE;ZZj+j+GsXg0N9H%S*KfVxa6cBSC9-M{UAv)15)t%@%(@Tw@7k7g915^G7-K zN)Y?O5A*&tpJr;mK@1?2El3EPoK?9X#yW&;+u%Y@FxDEJ(<xgmnTs&;m=&^F@@lZ)7MWq>rKwQ=YGYGHV3Ba<#3@U{lr z6p=nCY5WaApMhN3A}=Jx8r0F1$w7?B8|OeOjA956jy-1KRL}#H46d5H=Y=RDX&6D& ze;k(rVrkrXno-Xy6;^0Pbu|%)N)9{hcCAkco@9V}dlh%B)8zx}pnP&79aVR&*{(w6 z+ko^@U*Ch!+JW?&Fk200P+v1;)pxC1N-1wqmqC1{G4iM{Q)Z{wjOv^<52+7Xd$>^U zE8|yeo?%7w^5#dk3HPRbj=PEm;sx{nf7vK2>$i?0^{62WOQ9aLejy*m<*E%@5fi)% zAD#(&pb>MTUi?Ryxz_CX8zlFKG(;2fNGHZjTK9YbA*kf!*%^17M`lmPoDtoa=F($x zMtE+N|33Z?=NYhU7iOk=qrm3G86_trFUsi6G$h%6FcN>M#uAj{XIxTo8rm3tN@@;^ zqC(N&YZb%y#KSbi0Y9VOi{W>??7sLcN|pDTG}kPkEc%W&P$8S?sXiu4rpR{=>Fq61 zR-68M_m^fS`_io}1VOI859yRWoIJ=%3$ zZz%W?EsusTH2jpdN4mY?vAZM0nQ{f=AlQw=8T-X>S&n@eoUr;eQ&#T@5TXWRXtx9?8@@5KpdbhTE#UyCt!YSHoOROb@T)8AnbvC4&uU3WG z#*AG?2lgFHR|)?e)JvI9JxdwG-7bjKd`uBnVT?8^RB_ADR@GJprOvoLOB;a8*mpzU z<>nt01AJ2j`b?DKPaIP!7qKDw%ok+iNHcWGs22<(?I!h#heRGk;F)j2&B>@qK4)5n zbWHXH#-1pQ&GvqALi2FTT~j=BDKi9E>NH0}G55jcn_@dw`SM%l775|T>IYT!NR(qdFgGgr7Sxq1IZ zOPgPHCh=1};d2K;WS3a%VOyL`oL&d z7BgdlX1{a}jC0q5uTDHL-Ccr%Tlbh|fx_90zBnSHF{$ZSn@pBttrj=_&3LKFc!hFxsdk~L zDp}Gb{Mw+XhPzol@ddWnH>@Dv+nVb*193A|nwh2jHr|Ll$pX!kPk-KK0|x_giO$g= zjoF~%HJAG_0lgh0?aDn+DFjd=N%6lCUK2wh2jw;eg!Ffjo(Kti0WAtImSAU@SrMC0 z{L3y8phE7!rc1w=NKx`2PoU@7SbE zw{7i~YnH89wr$(CZQHhOTeEC)mTlX%`PTEqTJPR*&iMi7OO7vh+?kOXBchF~x86GW zy~!@1i$tT$$|V~0=^hu{5iX)axG#|&`6j>%VYc;jliUEOIQTpg;Ev@|6`+Bwf_WkC{`VB?|h_{A^ zdMDi4>rAjI|50^__XQWJ$T`uGAyny8LLAhil}3L)Y_rzosxLY2k3)C(@gR|W{O*Q&f5D3o%=5M!*iJ0KyRVi zIzcF}{yx|^e3zHZ*F?J|=W4#Mv_rE?qC98rgS`WS^@GJa z8S;J;p6c*=Ok<@|J8cwW#_ZmGP6}5zY1v8gKh(OLcje93Bu(jtv})Z9gaQ>yzHdbL z(_rfD6+gN|i#@C+6Wc-jK84hJai&E++f#y{GGT6yCBGj% zkK2Vfx<1G#<-qRh=;u}~5!cwzR};ne^wN4&iqD&rh}npg+9E6W#Q)7yK{PkJyv%8K zmGcDKb3XK-%ZDl(T_E#ZiHj@9e;Q#jG83@r7Sm`G~|s(u2dK_6d{ERCR*~+!AU8*t3}?7i&h3u$L)%Z3J^gM$2%jA7W2Oe zBYtE%^ZF)n>c~8i5|ir{GVae2e<#$CFki8t0gXMQ0VUcbKbK=x zJT-^TA3x{S`k@n^{Q!-rTmN7TYHw(vM3Xwu_imNJCjNWntTn^tN4x`6Dhfi#%x;F_Zp4k-Xg*MqQ`hp<@2M#;Xz$`_?B| z&%^kzVFr%!@-E{NQBncVQr?uYYl)lKF=nC%~& z8O4^3ED>eQ5KR`N)_p>r;y_7d6+ zZyreN*s429NFKjR^oK*?TT3)F?=48dV`>Qfkq>t7Un=QR3EGq@x~&{H2x+Y4u^*1J zD_g|o>I5IgUj8|vaMq|f8esInE@R~B?z7%qzFaTY-c!76o^8IG@twy+^73}I1#SDZ ze}JI6fElTw-Cg#3p7c^r@}c2t>1=8_7y;4nY6r=?8}KWdyrtKKCb*Z<@;YaKe;>!@ z{$#~G-Mu|Prj5tQ2^PfQhQ~T74Chq9wMBNDp>yH#Uf6FVj|!nwnxf8-FC~kG=fXjN?vlXi)`gcjJlIk_P<;^0z18b2k1dG9V zQPiS91rA_anO%0K(iVfolRI-R;vGE}-ZB!r%{YPKsTRrAh32q6&FpfKB|ep4nT-9u zoIo@(69j7RlulS$!XJ-ZJ`}GuJv=q07cAE5tet+K(UAaYc4T)epB2o7D#0sg$e&WO zl_u?#(L=lkWYxocz`ygajuuFl$xz9aG}2r--Lp|e4W6PoN@5QrPhUVGi%u@LO(L_E z>M-&VfJ@qd)WQTlG#Al|Vr3bfaq_F@5T+y=ucnfyta%7msgh+9FSi9=3`f=cmUsUY z9k15qOrrw;AGhiA>F5Maf5Fj*`jdT9fT+4ZVrB*|La4ic$JD%ct$5zybOa;7*r8h6 z6e~lB9Vl+oXM-peJ;%Zt)f`{(BKGK5^v1Kz@2>-A-*toy88&z3Zc~*r{}Z&r1~43L zSX{KDgbSVqaK{zpW#xAYfMF0eMHyqV^=kfQE|?Tp63(bt$?@PqF_~ZJj+YG)A*fze zRA4tpt6+Pg5^82)PSRE%mUaHyt250* zN;iq<)4hp{$8!Y&z{MtQ2`sfFc#0_iJ8asm)zKSIP>7&>LrMd$n)k)5bN)*~8uNfQ zX{vztWV;7G)hBt<$HE?!g@!Li{N+_KPqPP%f}n+lAN7S$l!jxiM7nl|Ost$Z0Zz9Q zmLsM|tXGWr5K1mg^Pu>ZxwlrZd;+^x8yMI=Kh{4X>!ISxLpDn6lRM^~^_`j$xHNqm z8B6aQAA`*OS)nw$QTN<#rF^hEgUt3P1o^q{n7&PpR{l|KEB{1aceub5uSsT!D`D{m z64j)ks=o%8rwooYT2JS0o9uK1%$8N_W71*~VP?BO*s3GC$|q+Ul*=rGL^rQh=*Wi1 zOM>HJgeBGdT{T3KvB)624drZ;b}hN6%Ia75kye?rGWi{2D%)c6OMfd%@`PlW!I{MB zS2oj8L)aL~z~kAb#?C7`#-;|L!>UDz?@Uci*m*GETtTb(N3lms%0~63Mp~&R(xZ!I zzQ9n0&`(8R-bD~J-iEqRsT8>?ev(3YKI`2`UKuH_|5=h5}DqR!;6)BHZY)bkAy2!_j3 zFep{kg0?Ow>00ShE~rb$3i;{-%|9EI_aR{6p_=Fpb)55qgcS{8WQK%D zr~IXh>GSb0f!Xy#W50zpHH-486s3I8n(~FR*&f_Kl3BuN1a=&;+x(zV`qkN`U_NRm zv$}5+WMy#TYT)8^ud$k4cX7Y{vb(yvu#Z})4@zqd!>-QKGI;swQT1uM-oR)HnxY1T z|E&tQ*Cs$Z)(sL-G8finQ%Z|PM}zeTNReg?`hAHlK@*iXrgCoDv|yf|He5gCsy^WE*K9Q4UV?ugweGWx+k8yhe#hUzl67`twC;i!ZLCQ%?ot zvCYl8tf2-)0Jc;QPnjWsU)+&{jG@`m+!!3Rik6Equ-k}S>yFwdW2yeymN5mhnJD!vw}ap4$l-&Q@t2z4BR!Au+i0WJ`(k4h90p69b%Q5f1s z^g4bDGnt-@_`msy;IB(r_@x;jOmTwe(!SjDc?kz1x%Uf&tY1rLXEPXb=#rQWgQ&@w zywIk}08Mi;{s!XkNWX}{mD@f{35&6ijIb_&Fho5mFou`J*piA29EngQWcnP**n{H= z4v@pMfKUMWC8Xpp_)mVg2}voNoC74i`Y8EaxP(mp0kVV)f^Wn!uVD{Bv*)XAS@9~@ zeRUDCghYN3vPE5w9C@$?@>snTL`~97L|4x(i_p(xPvZ=@!$Xnqd-t{vvSj^)@>!9w zBod7&s{kc)8mQFZh*HGz>I9mqY^QKkp6DR)@Z==5pUWNp`;uZ#QhRykOv{yLlG88W z`CP+oqO{MgK+BK!kq3K#hlP&obQ*6PjzHIHMXHnx2EQ{chw_*d50XU5OUkySv`)RY zg?q8!sU9zCKB{I295wJLPafuvDzpo3*Pz)dq`NBM`YI!Ki4;Ztl)@mU{^R1BJ?q|m zls%mvl4YL@X+LfEXLF6_gE=#$wkn@rEG)c7%CGy!;i50~eJf7EV-vKDmgcBr2U0Tg z`dIs_S3P})*TtonKPYOloP?~I1_!!>mXdYR1|dAoSO73ICS#0A1Q9@nPb(>gYg#uO zfk;=xBhg!X2?RzC(w@T*^7As{2;fD|gER4=(4AR+Vjp<$ep-4-=QFo7UAq^oQ>+Fs9oT6b2y~+=)Pv^7yFKLl9@Ke2^>s zv&x-ojgll;?G4uKO)}zawydbpA`Xw+^NiFpTNFD5~ zZz}lzHGu>_LP;cJ?|U1A0ErBY3hONeA5Rkpe_|p=kV6C{M);L~>+$0-SfO{>-9Ltb zhYhOt<4FbmK>5|&b%NGqB4r9do!1gD!d4AEy3H%aorwRMA#gW%5G`~!{nL{W%Ht4K zCmymMO)!Ik zrP{55GNuEXq(t^gRvwoCS*1mx1c!byf3Eq=Z)=9g@w=R~vi^eb??ZU_V5!r>FBcbG zTuWQd8IR}yJ)_Jifd01Nea{m{93Hyw-WpAF`h|qGxaQLE>Y=+liZRtCNFpQ9wlQgk zAQVNj1cjgPh*u>2Cxqs&#G?@-B+Bf7ICF4Chk|iir+~NqTM@p)^;2R99gr&CR$RCw04S3`to2QHoo53cSr6BEh z1jx%Ho}LsE%}Ee+IfOgGJ_jz-Mar?0hc?Bc)n}qyM1U*kz-HCQPC)u4|8oB;JhJ*I zPnJYACX%W_Qr;OO4z!ezm7bAgrwpN*N})B0+aoda2c&A9(u|lKQ6E%6B~&l;GW!2- z?E|HvuWgo|Bklz-R{77X^pvS&hJOzN$>YNewq3n3P&H|Ax6p!$_p>BN7+!DI7Z}66 z*a%H&5aHIbEXrWQtrri*73;6K4(U7wG`gF@1LrCZbaDN|LhSoT}KB3H^y27XLfjz6JL181Xse$nEBn0 z4~oRflQuQCVp+|HBMuhnox~gp1b_gw33yM?rOs*b5`N<})KQ@v(VFxfWsH~5St#Qtv;oT zlTb5*w$=~F=M%Su0Y3*Zz5N+xkkTy1aPzRz!-#tiE}3>?D6xJdr__AwMdh2yP3xTu zvp79c0o46kJ(#Gs_tQ%9%D4Vou1+>Gfm5~_+YfQ{5#Kl z7-lABdoEF&b1GZPm9S)%aDB8f#mT+1Rp!UDzW%AjPYgbs7>!f>;&eI6l1*1^S5Ys? zl>cPmNLcif?h1GxMvo$+0W+D2qh`T)wa{o^cz!}tZn3$T^I|A zZpi+UxO*9-vE%(FJ}_+VB<6xSb5xxpH}hjQB|cWeFl_;(!d%wVLPu*#?o`mwP@Mn0 zMqsX7T`7p>`{c}Wcq)0Q)~TcxW04v${y@1U?Vc;d+Ik_8&POO!SzFZn6j1SY`S3PB zEQ>3nr`P6fM4v|k(7fK)8x#*OH5_qmVrA!)wz&$kvm*^KGEC>-WtA?4Ukp(Q!04RR z>oB|$YI7Na2gC~D=Y8(6sZlVbJZD&S<~_Off^38SKqp4ez76qe>CZF3@`mybxGZqx zlt6Shpj!D)>xwqL5F(lelzfXuwOxSqw7>+vLWV<`x(OamD3G>473Cf(k!mZ?-Cvl* z%wt3VQJG8QAvHWmhw(pIrUw1Hob&XA(Khiq7#Sf^-w=OgvrrX;OAPfSCYhMywFAPpXXt*#2KJmWhn~WR&|oC!bV) zdz?4_$uISPx@Lbu+n#k~q1CQ=b@RIqV$~jbE_q&TvB|;e(&H1bwzoIQTKj;P<32gv)_8y8>7fmkJ*tS+ zbdp4?pd$qnLozQ$&e24S_u|!box0TBbFg*~RcK^R)#*Y_Ydj6@?Ya7x>Z+^h^38ep zc}>;v{eJj(P22Gmc=rZ(RHZv4U5e)G{uugt#@GWzt6EjJ&GvJo4RKjH`LSDH4)J%y zS5)>lp*p($R1;jRAD>4Op})hxFU$jU&i`B?wn=%*FOLTalVH}*@oT=&INovOVKmbY z3#w`z$+DRpjOExW4{sBed`<_U?Y7&H(+~~GZQ9>UuSJ@G#JBHfG@?&0T&FMU|EZ0m zif)1*be7|skTI0Dgo1k-1kcF+^d|dQ8UTN{rWajeHLD{0SW-}U>nVP>U27%Polexm zS9ueuRI(C2=JUeSdlU^>nRev;=24Y0TH9bG~I?L?fN~f^? z$qT_6CML@$E`K@y8fhF673~)`X50AR?K;YJig=bXF&5$xpW`EphQu5XU$D!7a z&q!!Wius!>0rFc_g=to_TM2rs-D?mTK3$C}H6&7pl2tx7d6P7~CKs%4WC*Z@e z(O3qZ0LQK&whqa_vgGe=9rPcUV?xa9zBc2t=bbw&s|j~{D=EQ1QE1MMIOe)=FS=Vd zT2CW7XA5}^30$9>czp*C5>BIHhP|v4SBi>P7X*CNHM4Cy{&=hAmA5gl1F7&hP^;B` z;0Y!~2ap_gJDI@sz0F7*u)Cn3x5Ui}ts@fm=*W$t5h%1>@hu2?43#pXQv&)mv}u6U zJOYV0d1|CbV5<3W#zVxR5`KppMV+gx#EFt$1K(q~ZK|{M^yyB7NYW(p&l|Mnp86R0 ztQ0N3d=}4toLqyfT1&7mJs$aLjr~4R(C3P<8Zhx-}^cB+u&C4kVPnIg}IYYtwWpXl83sYu9W zXfGNB>|CFAGP%JL@;MJ)pFNAW_=0XGv}rnVh|YFmU|&iFJ(_ZdFdQjUy5U~NY*P&d zsz141R1!=#{d2pv04^OeR-T2-dOhcM7<}Iljpl*-WbW@vqwC*_3!2Thu~}Wy@l{Nh zKmEf5G%0S3OxHyo`WV(`?RN9>HY*%xM(^~2H@XC~fgeB@qy(NqK#5gNI!s9=hG}pm z^kW*FkQ`F_IwGv5m``Ci(h%5lKMM8(H385VUz<=}d)6SnQ(0PWuVP5@no>vOb}k}SrSoCvelC(;NoC^k zW8!gwk=}ncr%h|niL#`l^DpL@QzDy6$6n`h=6y3UYlT6FF2#D4)FWv>V6oI~T@Y!t zmD4sm*~U`TA@>9qMRT?6t(vcl^NcLdIGTP9u8(J12^EqPh^qcfJx4<SnEF_Qr4~ihDaF&mo=)p1CL|tk-C7mq;GELb0=yz zjn2iUxc*jfqjAI#*JaeWf0~OH8N=k8Kep@xiMCm_N}YN}ykHsAtmS=%m&yv#e}#um z)%tHJlsp6nD?oN}3!be-mO0;)zou~2*Z(Jr4la+BAhNE&1RUx7q8V&0CSWd+B5qk! za!9CKk)S4ym9XShaLEw(T%6*tDRUi=eoVqwe9>DBk(#R(0pk}=;bRRTo~fC*DaoLQ zXPpyQ2Ucdhl2_r@G+t~-!;24~kr}(6UO$FGY-_oN9(*$H|0h8ZgsAFmNM51Don|B;E%VwuajSwz3kXH#4K7D2D{Ssu5 zySplnOYT25rhaMzn#KPoUMFX`D6UJ$-_AYMpe&~ICeG)|QqZ$#AF0v+0kwLQoTQqrlBq3-hQ z?xnQ{E2GOYt{mdqMOdi43Tzn^UFy>ZOO^Z;?4z>I9I~7*~RM#~5TyP4WtgP#Wcg#WiEe%-PRD zFgzMwX3S_sn(dN2h~lX`sB2hPNNQ{RR3*)w_1$g}Q!ZeRv0~~A8 zhe1%*iCDI*=&P$*I5ED(*U3w*fiIjR=qTyVr(9t5PHZTbJ!&n z%hS&9ZypA3ECBB4YtIr%(|ISbYpqiBiN&?)&?A?|AIndMXXT6*y?dY8)xBOhF7L7S znd<3GqyWkiAbN)EU)%H9n?nFWNQ~I~O5mF+069=U6#za~%vGfoy$c0sDn6~6 ztt!x**!bBh<~&U7`EuNc^M4FhTo-)YUdcU5*I&LDUFR4mnoED?L%a~h?;@D%@DcQfwMhWp z*!F*n`-HW;c2RR)Xc2lh=G~Ngn6Dbu4DuH&uds1PNY-qNZz34ivLgBNp^o!QhqUqG zv2zsbdk>oM!wtZEx(}h)P*170GY#fs-JOB}>);7ELb{cn4{!!gin7mo6%WfG2n+S`tl_ z7kCE+%j)|+_$WYrOS(AbLII7nPnN03*|(Mn{iw=)8n>#b?X zHdNH@CntowQrt&B=tKjZiZi`qhG5^fz*s}*w`ihJvlzcbZO$mJ?N?4^Ni=9CAlL8E z)TJi@-rws<<;6(6K!u~QFPI=MVKm#?W-P^ktEWyjHa2TVMnM7*QrtHX*r8!w_G=~A zE^*6kGh5#<9fo`Yjza>h{y7i%NT4xr)N`lK(aYF>PcFVYUENsNHkM;$d9%)Nr(k`T zo@01%&yIP0%o5LUgTe9yf9~)uzPRK2{qEb_E3%`wG4RS8(+5~&H}I6{y$zGcs*ioq z`@Dul5}Og0ZLfiT_fmi|pyv-3HR-dPJ>XZz9QoNF4Tv{cQe8Mot|Z_JC}j|@N*Ebg=JlnN;(^!IG_YmQ$NHAmZ1EHxT8d+JZqF2_MoJM{qMpS z2bkyY>2!kCtE554qfu!xT}Bv(jWa|~HIL*e?7c?AX@RIICwh)+Y~a>mp!}6_0qEy7 zfTi#PA3G-=HjY)?A_bD=j{HM|8W?O-Gkt-dq`k)41N|t`I3eyBUwRA$cwJrp`G&y1 zj3w@vz9^ol!(V8mghGZqlF8<|IC7kIr{TCE{q&`*34hKK+^bjFU*Ag%MkSK+{WXMU z@TyZ>n`7cP*3ZheKP(MrmPPW=R41&>zha8Co8nPUHP9!p0|&-?@3=%h0t6RqS(>}u zhnI|3lnAw%6Fx}ttK(ctaT-}sWQu=aqcKTTw#oQ1&CEBJa3k9ymKl)d`~pRtaK{R zu+j)_M}ylH+M;sDGM$lLGn+F5vedNNs^!-UQB6X}5@4Ho@3ywjqUcyf&_cI-bkKgV z44`Fd(~GX=N}NOVL<4C&TH1fC2n(`v%U%bsoHVbLt^AB|cq!;?Zd^`bn= z-e+Qf{KQlMNOkCX%@Tlw9IE5z*IOW{E^vI$&pDor6exDdHXpt{brJD05+HMMM<{6U zIf2D$`4;xF1^>vuw<4c3o4NOboM+FGgu@YD$rsHL0;f1F&8QB9!|x2j0coe9VLDjs zC1x*7F4CKSe)10eOI{}`)&lAgG7Hq@(*p`U>qul+(D7>5$Evr?kB@?(*<8=~U8s)c z3iDDJ%`L=!qjm7H#untC&FO2jE)G4BRIg%>LG6cwnBodiu^+?xUhX<|ZG2)1B1iyQ zuo4&BCS8BFF*YZsU1!3o*CwccA&2MriXVg;)AHDh<%sH1n`-=rXhSbj z#Sh`i;cj#oN4j&=05?e?C2R3p$Cz|@6|wA688HPjrdT!G0?;xWK}G5IIAZ=lmsRK| zIlFt-A~i-A?24&>bnoGJFn)6!2q!Thll*1^0dj@{E<>~&GRO$WP>vT}i+#Aky7|U- z8&;u-<97OHA`#dn4OiT6AKu#Lw#^n$o&H<4L>-pB-o@qg^3P-pp}%1U;NL^;Zntr( z!Oh>NjATSQW@uZ_rlq***ns*Ff{bA_T(Ivb8EBVN9i-q2j?PD+hFWgy3~sxVrlwv% zU;{6W!(FQnDk~1l9-#b4+y1J>l{KxU5Z{UK62-go zr8KTSj{5BOV+N_5d3MVTOAQDGtiRm#xQokYT$fp0TvoP&1X%NW0K2-M>#j94+;*lF znQrS_K|NO$FN}*<$2BHEoNpsd;62>G+AS-GgKs#(pCO>E( z3+XET(-kUIWl6$e=OG4if@|tflD*5v?{qU#DOC<*yn}V?gle&bkCdr|N9o#xh4V|*1t)!RavSVZB-hYlQ?^Yh)C3Ek|TQv_~ zpy!_}J^pou-hZy{6|qH2dHJQ13u=h$i%+(_>jopO2sKrq6we12Cm05=lMrjvh9l}? zG!}Odl3xlHdr?~zwvz91h{zAKc_rMZ70#1fcNJ1m>x`Lrcxmmx$C#RN5cgB#!RzbV|0u9j@T$bW1#2PWas3F?Pt@o}_ujdBirn zl_d*Gca4%;aX%%BPag$Wn}1*J6p617FtBl=s(@>jGWm_Gj`QGRv%Hobl;ZZ`d8RRS z2^q1@Z6eA(X;a2uLW*(5k8~Bc-vPh01XK3^m;M4=` zloloi&gb^RtZ;*$r>F2k)h+I!6S+_KzPk$WIu@ULvnV>BMFI(CoavY!8t>|fIbbw! zZ9M+DZ}zAy9y>Jw(<0ii3n`HGQ{>?p08WXZrc^3#yGk~hXllP8ERX>9;ovN>!olL= z!^azW;Fi>vijAbd1RWC<+Lg2~=uefnPTOc*tkAY-*p;+J=`Ok<4yL7+0PldH7bYgI zyg6dgP^;_r3{8VfMq@7Rw+ufcB3H7KNb@%iNQcxmgp7WvLY^slzyM|9=1A1EsWit> zKl-yH{XCqDvbUi1Ea7;fap@usPb#vE><;OqkeBE*(2~T8d_u&M^pz|MLtGNA^E zk==i#fhI$0e%`;*fJR2{BHam{6wa_MVL$;i%-4Uvum?9lwGQAeRTXbr=bNJXU_At6 zI42lA`P2YplXD0%R+RXu9$vDI1?#C$N*A$N15upqM>p_is2O?qT^WhO_do<2%2*OD z!c!(;gJD33Xj=R5cq|$-*ryTMALX^yS5)N*EBd<&{>e04-ULHNQplx=e*Y+>~b_GB8 zT3q=loA*3)xQsJzsBBwfXtx+YY%YX9aR zCBNA^_`b^_KzY=9i`DsoHu#Zu^nPIN_>k^6JHFTa{Cyp>F;Fw=X5gfaBb?eBb(@|4 zakGvlwc)shvZf~1`{b95eKxP-;BUtqMY(cX!y=jc56rF>Wi6E7#|u>aiAzX&_Ys(* zP))>!V=w6=ZH+$Zj;d;H&E(ziLMHM-Rll52040rFGHRbBBc*wadtNulYEXKkF4PW# zrm}m5TXZ&opc0DiJib>oYliE(GA!f;N$<|El%l} zx=DG3lNScJcD1%W^@Lqt`tdsDWT=$$Pv4VQz8Opav_=tJ%n-^J*roQON-y_?{tW4X z1JXuI1-g4_Qcy@^SXCn?UwPoZx*0fw8zyYsvi8r>Z{(}lwgogS1S4K_x!(=2?X3CUGQ#=^5sLDJ_2!7JtGI7Gc^iaWSJbc4oB2 zx>^C?A*MDfjbj#ocOVE=iAnRs;&&WM<8)&?0<-{5`^4*Q!uYs1xqfBbLhAENCoXw^ zaEk>^c3GGj1qHF5lq70tkO1H&dmX+@ZjG%b%-qc^h zzt}oCd*S-%GY>Z{k^AW_{NX`;#Nt8aJU74!A@W#&&@_}bAK7?hYIHC>`R;3qdpr&uezH8K@Kj|jvH;M?ePXYc4*7OxpIYZ7ra z$Tl*e?FfjouT}3sx&x)F57kZ(MPde|-soIIi8Hyk&mvtzwZ%@}1S%+XXhx?ih&i{- zfH^^BHy4*zDc6Ux8WmCdj)P-uh8@FLpxo&LZ%SCZetIfHL<*=efKY~ACn3=PjlP?!Hu+N7m z%p<2CN^Z+|v<6q-4r-#L>3#jQh*r0ueqLjIC<$D`@T)>4D(f_mrbeggo zC(w_Whnp~;CmHVZOYsjtT!{_^KvCM`DtHF5u2%CgqcBv&?MV(CBq7nPzDlKyEJH`tyxzXx9Q@i4mV25gF?y~o5gbZ z+-4~!DAr4uTMb$5F7wZR!C^qaryH08$)iPZ@|(T<06L`P+H3gb9x`TV_ulp1@2VQ@vY!ZRHxTW}h$O*JY;|gVuhWNStWF!{1S`)(63z zH+&DJYv4kjvL+&vCLlayfLkJfurR#QHcwE#iTNgDy7UgQJ=*#J5=W652D{#bvi+ni zirAB2a}QKFyztNO=Xa-V_41h9iu+0v3;tQq60ZaUzqjB3ov`%u9fbX^?Ta^_AglqW zOM9QQ#C?EG!qU1wTbJQ%WZHaZ=#n_97EZ11a-uk}3Cv(Do1u$kr+?naOsQlYO>UYA zOgE@`>zuo?HJZ<4(ot64yb=xkbKlIexXW{U4-!vIkH$jUxdKkT>vu1 zuf9JZ*f;Z6^d|trxPg+}b#lLe&=s(TQS}p)h2(y@X{j3l;`|A$@Cdewg))?Co4;ui zphu2~m)=U`r}udc?!TSs-<^RAef{(Ps9K9d3L3Z8^F<-@O96d2OS@v?hMMEoaO64e z`-J3Dv9W(}4*G8<&XQd2r z-7eIqyA^B^LGFwsBKP8hvNe}zt-px;6MZ{EA`6IR&H`!&0BT3(w~qmc1%oC|1>Sh8 zw5MCouGlJ~NQ+V}Kk+7IAFU=mQr#^7GK9E~DOvnT=>*Kh)2{!9BW+aOH4{6;0=fYH z|KMAvix8}=tk#YU0{FruI7H}w(Mqk~Y{1>Kp)NnSYdOJ)m`tghkWR#LIplXdhYoK zhqg*Gz&-wE0|SriM9*n=;`Daasa0P$w_q{*1 zVs#iY^3v}^E@M>|XV7de!=3(fcn=~!hy?lcaH(@XU8tv=q=EWoW`nhnVBUhb0lTDI z64`>pR(+_DHa%Swjz&E%;PUixA@@V%Hh*C9r=eo^Wh^P7_}coAEL^+?)5dJ@3kuH4&#RlmEy^M6*7w1dDl*@zG{J2 zcT>ga$7wBwNdKh(DHjYvxQyfNlgvT5!dvR_uH~Xdkd@B8=4=7u(Kv=~8V9>7T6D5O zZpn3I-CtCiM)@c)3hFz19`hh~l-^gmk^93mzsFn)PP^bSH&HQBJZIMtb$^*&>1yt= zMSiCE6l{CdRPrNVVeHZ;_Ob^v<=Rp2Wk@2*iIW}y*~gNnf`K{5ifL2FS{{95{jl73L;^bEX3I|4UcYvUAMj?J?fdX+M0_U> z(ZwNIykk94Yv2Cf{;GD>%rix&o;iQk?1i87N%w2WghTTJcELuTi||XSHiSa-Jor14 z;f%j*_IGVXFp2O*c$9Kz2@GQ*Rot?!L}N^2QL*VCQ8_k#=7sOyTC9O8k^s4LEgO%PgzB; zOqxBAbGSXQuXF4J@Zqz=hSVdt+rJO2-vG4LBT0JPzn?V7fQhD~VmDQRgfqs2O^GtM zV2*0YsDqmz)l!d9Q@h&qn1zw@HRtdCRcxfY!+G;(VO+J-%E27KeT*sw&Nk=-MgT`m zR#o7n;~bNd^TJjjA9H5gPlp#2v$Le}rZ>0rr0_;D+%ayilyX3J*#hh<&4vo1fpE;( z*rdvezM4`2>74Lq3WPK9hS(7jFiLAJ0I8dZH<>&#~gQz*>G{R@Oq1v%Zcv`lAFAnIZ{7H;}D{_0HH?WUZiHr0k*tHe8 zn`j>=@Sy(VuNeoe9 z;SP3j440@|X^j3RaA$+rQ}IrAae(X9ge*#D!`erqN{iP7r-gw@=?*sNHJRISdOs%& zjfo`a=Y!62b<1jAZg+qW@E~wVhC=+==H*`1zH}@yueEA&OEoi15wp}r~Eum`G zT$XSbx%V=Kxs<_FLBG-kUjcJ*yPU^gQ{nGXo7beRdv(uI((&oJD`}{=)02aphtl8P zftHxJpS-8&>0|O(w(K}p*8A%eU53FAF{GjU8??`DGxd#-CHG|?W-1>hs5wnoexnqQ z;=xmuEUA^sFt`ZW?4r7BI~zec0j9`Yg*zCa(^QOH^YBLQEN&*6eTBm4b6LfllnJiJ&SDel?Ue;H{>s6obM$xTIFS3wH&F^)IfPt_)7~_V zRMs|bCix#?eT=fIV17B6Y6`)ax>M-^9Q$BTy!fH}`TExz&#>_1gBVdAOg@v}@OL;281&t(f6BB5HR z-4N`bojaj!L6WtLr(OVVpzF~tP~b1kyV&Fxh#C!!2|4`RD%0H5n7|`fA)bS|1i7gc z)|8tW$FnriyrSQ}&(#sfJ2R6UYQmKqfU8zR)4%^h#gnpZ5DI6$IKt3Q4_RBMAdb!l zQa}u025`Rya>NrFuLdChfnp#R3OanMoeCfw=&gPk@`Mt=dmL)xvBg65%&K=26Lf=o zM+VRpN{%8O_NA&!L!m@sjt;4XT#?zF#z%32aYq2w3T-Rwtn^e69093Hv14&_28y>W zW^of>+y8?Ym0_1jmEZkxLepGr3P}PJl|kRVdhH4&`N$s*q6?HkT%ElQRuUAunZ!~) zyjYp|CqHLqq+EMlCR#$~T-hO_s)qS*-=15My5!|f$gNTF;I5uMSAEu@$)A07=f>o< z9q;ym&$yc7=To5@H2in$qtg#o6+NpDGMJB)-=o&glSCd3OjjLiN~+$9T}a4P%`1nS z2FE=bGP~!pHutJG?-sUSO=MdN{nuWK@-H3z+lfTMJ@=x!*Xn0cFYh$?@sVBhZjPWL zF}6DKKP_pBH`sdpOLqnL>zPgGZY)k;RRYH-95pkqpa2lIQ7zmz)Vw2Pk8@+64d{{X zB{b?4(Br7s*#^_40hCGo+R10dcFUR#l84iO09sJ=&mrYCt($c!Qh5mqnunFN=mK&E zn{W zHv1ukBXNsuHWeVXosq3MN;{FlnjyUu0nI9UXHoo`*RZ-&uyrW_F3GwhrS;7q1tk-a zLxymJ^Nq_XWAeM`f1V@?JtLovDL9`T+PW@d7ZC@U5(}XNG^A7}nEJZBR(XhIvCp#j zL}IuyZyaSiZvp=ZWwRQY=mVNXAsD^pNkjvRhZ@D85|);-vo)#c#^8$!S*MePsT5&e zgysVwm}p4fvx_TSX3yBF?}j2%B)Z=|eeGZUXl|K2M(zL%u5b}8890k2HDF!JpmZ$S zZ7H@a3G1%X?bBJhSPQdNT*Ht9dguLz!5byPQl#@$q2mXD#$U4G5wfvy{Ak1rIEB-& zVXL5&$SxMAto&ez!5r~W`#U-n>0cv(Z=cFWWZ@Qs1#>Hm*&_lSGS~{^f@J2Ld}v~w z`o?B92Xb19y6xaY+-RQZh7Eh&rQYI=e+H%H!)R8oRJErx!iKyMnd*N1kBa2MvE|w8 zRAf71l2kCRQ*dr~8gSS9hlz43ajE~8iP92hbk!_2fmDztFFihAQzp1e(FLCr*qhC& zZM(s7eVcd(l7odPFW4Q-^6*5>8oE#PMO45VKD z%Tqy&nc557Soc#XL}wAJ^L_v)g>|u&e<{mdaE9?>E!!NZ9LUn|GTjpm<=jk;g+09r zL@AL-_<0|Ir=6BLw zaz?!kcgT{2-v290{}*iik0||zwKnZM>9V$bx3Bq}x>oU-#HH*ud)QdcTR(25%sOj| zY+GB``(7?JZ!&nejMr);KS4PphQ&yu#X1~NtxBo#QPhCQr0tlbIr0wx>p^ZpZF#U(3pec!; zvhDF|wSJg7DnV#}1$XBZ-RsKVGz7l#`2UMqvzGpYTK`V@wa5O8T7lY<=s5ouYDELM zQzJb*sx;#Kx1QVjm!pCgD|W}@e+s%$(Z4M-n*T@DqQ_TL{#VuN<3lNqnl;vilCf5x zf%>h4u#HTlr_aE@yQkK3q{})yQT)_(RhAO(PixX^*r=gjm#bJvoR4iER<5CN-DqMz zZL;MuU-naViz{3I+)=82j#(*>X7p6M-DGwj4{~~tZ)@4Sd|IqYwC~m^FoqP7Rc6W)fwPx8?O zY`q0X?SvFUSrf7VXd{E$vSgK~VQqkHJMv;v+8G%6df20e>^rMAhU zyx6(Cy&|o=r%M{1(>n6<=Z{`_xsuBboZ%K)m7D(-POB&dwI9dg4yCdoj;^E?Anmx>zeUP z;=9Y%!Z-M`Y$Q~?NQ#GNd$qHIC(4q%8_$>cbP{30P{I-6O-%9J#n;ltw}0%JAJF(; z6YVm|pNNwfqN($7XZ2*gZ%r=6GILEf#abW|YLEsV#gO%61fMMBrL3y+77E!=_KhzP zO~)D>igv}6Ux)sNMy_EIXL4wXqw;=x$n7M8%jMx^mw`qHLTUXd0-OsWH7ONCi%YcJ zwEJi)(I*?L{Q-k!k@=jVV!jY^-*w05DD8*|ii;g=G?PDjD_!uD zePS9|dU_Y?W!c2sp}J+%(%YU9fw72{75<%Q7o+*8MG#1F)3{ZO5}w#tZ87!HI-^ax zG2xt{mi2Q~`%EdICTWZL-1bLLMNzUFlEHeF9eOk1s#oV~BX8KqtX9$eIFSVRfymy` ztG%H>(Z)I<`KrI6H#BXD_ltPc}bsiKuwhE&%!vW54}UW%csYU_Q{I@vL^ zQOv^uCY2`e(Q4A#pewVMi7PTVM3FBqt|#){6nKr&WTG>(uVLFa)P-WNgEh!zVVvkk z95dbudPe*|C!3CTBNw zA4NWZs|{#Rf=q0fQ*bjg|Ag;CQfM)dNY6lG^gb_&6sGYv2`6w0js_6mBRbme!mlEq zfgP+e;wkGPF4pj4JhgA&5@d1pS{%eDGktg8G^{c+1XzW)_clB3eZQ)CSrKy#c(FhwrqO5A03& zuYcRbd;`_-r1j`7<+{RQX%oSctG6VRAZGZbSryHEyR`m;4g4UT??;Uq%<~@$s#~Ot zri6j$H-+P>Kz!5BWAJ)_MM6NkE2@1E`V4^ktv`3FCZTb*Ie_W3+PRD&qrYH@Lz zX5F({YkINfoeC+F+a?x;Rb3JXZh5oDDL|3G^1bCW(E|%HUpjuZk#WmdpV}mZSIAfs zH07<62CraDQJo3$`5kDUk8q#3J+R~YM|RK+J{&{zl3a37N$vjJWNIzeQP zYq|q4wb!jA=w)QDeSIo;?#_ywqz5W)8=U7kr4gqg0ge$3#y)ss6K-WyW)+$;gE;cx z8-UqhKB|wQQGwJhc!orInepjc07eljoejs$tQ_}`gGH8Y85^(o-w%m>ZwC#j&AdbV ze&}h^>-=QyMaN1kTQU#T&K7{o!v_(ne1|VHBtcvz$JwUr`>vW52rcD7-v0xB01pGp zos&Ko^8DEjx7nk;9^(D5k(H?2dVU07e6Af;fMy|ZrD>h!M1yl$i6l{w{*%mEQ2lP> zmZ|&F6UF-Z)FecVHYt7Yj6C}C#{+7xzUKtrS|aLe1us5Yoo(*ZO&VJD_3o$T7S4Q)!g63qtg+C#7U5qaKXDD zqY#*U?)KNPowCu#tH)J9?odE*!$bRfa1q^osAK7wzJ*e)70+|t^cFu~(TMs?hQV5L zo^eF?dBtS|>x594i?eqZXp55CUyKZHr!Nbq+?(l|_K(FYU$fFoN)XC0SCw*PRTFF5 z`Lb%r2Ax}B_CtzbWGDMH}?0bg1Gg8Jh9@6qQS*O za<-oJE{;fvM1K_&S{uR6b9Gjct6oc+tK(e|~*_Te#8rcKx8Jf4@V{#UgpJQCz z+EiY~WQ7R+0X5t>YtR3S4%gM+`8eK96yrq0@9}l1C^PggF0zksj8{@ zKi?zq>dJp}Tx4kN@{SZ|^9Y}$4~s4iYSoGKJSwUAbXE9A0eg^taCSoaYq|0j0!!1J zYMjRZCc(XX z!Hw1Y7-j|Z2@ReqaFe+BB{COa-$iU!?EGsqvUoPfB2K%Fs4TM6i6xD#(OV#NAR=Gd z(4i>B&uM^@y05{QXhK4<M?Ym&h+M#@1SKU04!p#6=|6EXBOgPoXKexY#uo?{p<_w3mVQRN zo1aW-@gmzZ|1odqdG5W=Q%x2OLqn%;QUN7X&;N0Y-fQr65}OyTnirXKel^&}=q0&W z!5`K?Zn-9S{^KTOOdk4z#VqB5MGYGvcaB9C?Pq5MWD)H2>^ucz@E89I!-9@*_m ztB@3=4q?sWXk(w|jUhUKKC+vBsKol={=XS=9qf3^=;fdRs(Y@+FGhp>(@6q1xO=Ex zn#WN_?I@Y_U*DDw*1lu5BZSTX4rt9NG4%2XT6u6DTPJNn*JwyfBc6OU1_Kr`De|Dg!9 zDASi=?UsRCLbW%JY#*3Q_&#jXjQ*MPhLkQ2&}G(*Cj1EyC0W;dHQWzlZrZkF>HVR)S(rMN-TYy2UKn&+C3U znXSHz#LsrE&lJ%0aY(o7UGty1oqXITmu_0FDBS@G;Jzba zarTDU(Z{Fi?6qiKBGONFqF;i2i1BGUb}S9=_}h6E3&pS$?~z0*!))Vh*)R5Mh$iLE zd_8F<97E!3(njs4v({h)*?tld5-7v%`@!>s*KeS~yDV9t`(*^|Vx|ymy?2gO+X3Ir z#bhp$a%)a@r?v^jW;3mPLeo8nAHlQ!qCGMLh}fRoNJGdOaFCtiuUj5ngG521;N=&r zb8#y`5NJIID%?gEnvtLIUk^ z6N&E}VGm#u`!Z_)zr3pF1g;W&GvIYZI?vs6>~1R;(YT(uEOMyjfw;rn@S{lNqD_mW zty4F6dE>+Q?S;>8S_-a8`Kq`SG9(cZuBHj$5WgQ0L?DJGR8{R$So z^8JGBoZqIqX-&_Z&*!$b(C+-|y&mp&H_zexL`2sl#e_ued)kw_VK3Krch|4(QXHG_ z+XlHheAsy5^V9H_Htea?%lVS2s+C{+YUw^_f*7K1^Ae^_k3B zq}~zzMD@W)!X4RL8}W{QpGJPieCRE2;Qcs93;%aKR4&;5iR~Ua$aHR_$mKiq#zxhrfNNK}(>jv}*ifL&GRCxnA7_Y)VyWYQ)pV0IhV0{B|0aQcYM z;bHN7+*6cT{2#cdm5k_H#tKG9$^e~1>xhJE+zLzMw}&KPx`aVI)EkCWR*bFWx2T~m zHLs0~F)&+0-$B00#wZA=jT2AGMGHhUterTMQJoB7*fT;DPkWa{FU-IF{{ZdNrp|x@ zA6YUaS!2f{B+T{?GtaqnW@n37zC{auxy9{u+|b?8ptvl)HjrTJSWW3DO;M~!Rv-fc z19)cAoWuHC7)9dr2d@8r8at!>xGATk1*h7da+BOZT~kIjMIQ8-jU#JtnxUw18zCT#pH)YEnzXy8D<0FHaMTh>w(3 zk6B3~%{1QgI+N-#nf|#2-2MrcQU%5gCa6a75@}c%qY^7UMW({Bt=q8mEU>adN;4ig4cTsct}#-o}D5G zR~f&igG|pANHGU2EVqdhFK@bFZ3VNzHaFtmC z*F&i9$>BNN;QY|U@&MaTjlI>fem59{riFyGQvS|gC!0+WM_w(a0D^AxBu>7Y2DOe4)6xTSNAj{6 zv8G(VEapY7jMS@%4jjQ};70Sq22J&$jD`ly}Bi z{cS7Ofxy^ULA5*r2X_ymkr&UcoL_+7IfH<|L$=Ldw3lX1D=W>2vCz=DldKW3SXS-F zWTf#sL$EW|f$BQ!5W29sP`_YRo~7x?gv7HY7ix1}uN}vnMV>Q_%l8I|)XOK$A>J0Z zLnV3qoM~JeaADjnmCMJxUC~{$q{|$94g>CABnx?R`?u=Et&Qv6?CaM<@%Qf&lcS0y z`Qfr|3cEYoJDV7P-Bl`myXjk53Cen3k<etQ&4o4V>l^h!*1x zY|LEVb%AhIB{}tcWyrkii5yyfe3heI=2jfL9i39gd4C_^gQ|G)7`7gRDn6K8r=ZI) zaE?O+iT391@sJxL&>JZ`>i~dz=es^evWc9=m12Gm}POc4oWdI&yhFx4u z*X-u3;rqSk07F?VF}|cuENjs=`TP|*O%5 zZW_Otu!+L?p+l)OuUE= zTw%Sq)-RNTr>#3$_w;rSRCpo8BFYCZ_~FIh*Q?OpG-mMb%NYNYt5jC|N`&c#bBBpK z!Lkr^fmK{u^^wJ@aw8DvyOShTESwSHr;eMJzf)|RB*(#B#8vP#CRb+CLR6a>$V4vb z(9ktv#LQ6_ULr^he@e_T_grfwk{Ie~S0a9_AE9j1MjI8VMxSy!5K9qdcc@DGxuZ3* z^E!CBrg{w1t=-E!L>lg^lC!GOZs3@`N8NjaPuGW z@Rx9|GA%cqHLmWM=qE-`$dao~tu4b*F{JkW!vyPIb7O#FQfa`n)Ks5oxmN!NBCQMx z**`5GTNITl4ZZCA8|t7FqRkw0@nnAs19?vU4-{H_{WG(_Q6!uC_c> zuxnUwB3;+1SuJnmMqr!(21mSyuzS*-NwJW^GRM*CICZ3DMx@AL3Xo=uXC-b@m_oA7 zKQV?pP1X--$E2BxIU^{IU8ePXZGF+|7ol5epxaPL*qwO;>z~^3Zr?tqB2z|KlTg5M zVL^7D`hN{i%E00NU&gVST@`FeE%~1Op5CSigdy%pxobMNrE19Sp~0gdRBVQE zhE61M7OceRQaw456J;bC z+WKqnGZwfwDFv|J)NwMx>i}$y>_m9k$!X7A zK}oN>%3c|cp6uJ9?po8 zrxCA#ESj3~uFNV}W6$AX7=*k1vnzDtqic2^HAqZcXnvb^Q}ukDQ_A=LZ~ys$pLc-! zr59F*FA85kJMtZLsT*KR&$)BuH^6x-NmZ+9erEe49MxT14(MZ#;yN!NQCgMK}&euwX%?FdWF3#sAm|tF zOn1>JX&Z2Ez+ViKe*H6pLqqGquwWEB0u@~!N&6x5#5{gu3l86PCp@jV-{aA)X-c&2 z7MVDAr0I9H0lGt)wFq7Tlf7t{zC^js631$1iNj5cfN#TxgcsThhRkfBWo|*U98YPj_)%WRA6DG~X!4`Ye16#z9>n4#)UHGyzWbIvOAmtp&~#rLZU zxrBGiMGZ|xs%kAHjnwrDXDWeLg4q{?nF?!CHCKW;5o3dCAQ&ah6iavJ*g@iOm0+*; zh}=eYuj;4(VNnSrHa&%yHTtg`$Rmh}T+GI*4>lb{p!Klph9!}F)UXxaCEAY&CN_>w-pR$rv(u52a_v;NF)qBtJT3&Wu8V&z$9u(Q6}O z9{3w#^=x_eP=%eTWOaoTU2bg}i!#Rmk2!XRH~VRl4@4{Q>q|8z2F_=Jfu&1+k-{Gb8)D#TS7?0ozejHlymi==EE`;FtvY6Vs7)yf;SNl|@Go~oe^(h0G*ZoJ z=AovYr|X|N8erL~W z{Ia{Hj98a0zvl8R(+v?qyEV!%L{TTe>jNs*BLu*&kOnQjqb)s{=S@tzzs5ZbcmDPD zoe~?|6#d4atP?ToXlN(yo6<`^ciIPhS}kO(q`nqA64+45xS9jV{zOq6|5>u%>2w^) z`Wx(B%M!s!cPD!ZOmc*mAZ|F_?S&T}@ludquc!W=fRI_#i=742ep}>+S7Pjj$1D9AO?&TD0GApx|kr6N#b&~Kn(D6XH%XZ%?+#_~XmI#9+V@v~I zx|>>|p9Vm&ORp>RNQ2S~h#W+B&+#6&GK*B=`zeXOEBP#T!E{HthXTtnE>fcpdXSh^ zaDG(sUaUrN>iw_J5OPyuJoNQ^clD6(*=vPXvHn?ca)MHmP4gdmv+twdJ+Y)N6M4G( zY|p);rHk>LHN@^xprYH=K8RN_U3xR10W6gM+UJV`oDt2M-9*3#*V2T|UdP4e%bq9; za|pDR6MpN}y2-Nf8x+p`r7~ao4qE3l1iz1LKw#r7z1p(1{%=hYDTY9 zqAM<)8f2$lm!KBo9~-4U=m@py9FzM_Y*Z2ARHm{=x3sDj=qz{5p{R+iob4l|t?%q8 zuN-??QgiwEKiVj-Mzyqrj`DlylZW=J+|~;5FcA9b{Os~%iCD#bm~uI!qQ2S))r=Sx z_>4WWI%21kEVvkOe`^v~7bzCx*)v?!o1&0Z`LJ={>#V)s)I19P`Q8ow=%U|cI1NSPH)T*t{Ha$&NPnC-Pa+ zy2BNMrN~E?>!Z_!OUJXvC8m(i15;B93^kBlw77moR=L1;NpH!s#=kyL7vX*_eVz=B zS@2I*dN|u=5wQK1xgpiM65G_&L`*{h8xY>a8F7L7lZUxGn0^Km>Ph=yXOtO_>0KE;{b;yWI1oPqq>4@$0#8$ zDrHYbU|L|edk0^?9l;dw-Xf4@fCICRcZ(Y%3R>Ce4gZ2?c7=W(>~4pU_hYf6{T^32 zI({gGN-AT82@#$eb$qak=)6{|-;n?u@?g~ibCiBuf`KfCkyb>(jM8M;#K z;b0aAkDByIsY4wBbvrt=mmiaxH5pWWtt~xwEgOByO=^l;s0yT;k_+({e8Kiv5_-*; zzaJ~RmKXD}`t7`^2Jz76AGqxl8Ya;;73m6Je7TE1v<8ncZC8~kK^f$?5UrIocqP(2R zDy0>?u1~K^l=l9dW=?~K*#LQV#){q7f=9zNK%THs;I&HjK*1aId-rp^w*_cgW?~%8 zc}WG~j_4kTPXs=1&94U)RPv);Gru*N09UV7uJ-j&jYSwq^L%K^E`iY#5#`%I0E99dKjgk6 zwF^qoO<&CWh~a3O2&)rVj@myNC$kGKQk`68voMbyp8I@nIZwjdLw+pOBRgWQf4@jn z1A+~)V#hUPgdmnLT- zQ23w_?q?dgrI8|i1F?YxRPIlaMq7ysI{WnVMO4trkt-Mq92^W*Dbq;?2L!_X59ZGc AYybcN literal 0 HcmV?d00001 diff --git a/helm/pgwatch/Chart.yaml b/helm/pgwatch/Chart.yaml index 957c94d..a7efe7a 100644 --- a/helm/pgwatch/Chart.yaml +++ b/helm/pgwatch/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: "3.0" +appVersion: "3.2" description: A Helm chart for pgwatch monitoring tools name: pgwatch version: 3.0.1 From a2cacdfb9624ee807cd89b694a64752735457973 Mon Sep 17 00:00:00 2001 From: matthias Date: Tue, 15 Apr 2025 16:24:10 +0200 Subject: [PATCH 26/32] remove unneeded file --- charts/index | 1 - 1 file changed, 1 deletion(-) delete mode 100644 charts/index diff --git a/charts/index b/charts/index deleted file mode 100644 index 8b13789..0000000 --- a/charts/index +++ /dev/null @@ -1 +0,0 @@ - From 0d5cd0464c37a6206a80d7d55e3cb52381e36dd4 Mon Sep 17 00:00:00 2001 From: matthias Date: Tue, 15 Apr 2025 16:25:46 +0200 Subject: [PATCH 27/32] rename charts folder to docs for github-limitations --- {charts => docs}/index.yaml | 0 {charts => docs}/pgwatch-3.0.1.tgz | Bin 2 files changed, 0 insertions(+), 0 deletions(-) rename {charts => docs}/index.yaml (100%) rename {charts => docs}/pgwatch-3.0.1.tgz (100%) diff --git a/charts/index.yaml b/docs/index.yaml similarity index 100% rename from charts/index.yaml rename to docs/index.yaml diff --git a/charts/pgwatch-3.0.1.tgz b/docs/pgwatch-3.0.1.tgz similarity index 100% rename from charts/pgwatch-3.0.1.tgz rename to docs/pgwatch-3.0.1.tgz From 4f06741d72b5e38f26d33d796109ac8116948b87 Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 17 Apr 2025 07:43:10 +0200 Subject: [PATCH 28/32] added sslmode to values --- helm/pgwatch/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/pgwatch/values.yaml b/helm/pgwatch/values.yaml index 6f2f837..9517f9e 100644 --- a/helm/pgwatch/values.yaml +++ b/helm/pgwatch/values.yaml @@ -19,6 +19,7 @@ pgwatch: # sslmode: disable # username: pgwatch_user # password: PASSWORD_FOR_PGWATCH_USER + # sslmode: require prometheus: enable_prom_sink: "true" new_prometheus: From e3f94886c0cad3662199a3cbcde024c7cfe17e3d Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 17 Apr 2025 07:43:36 +0200 Subject: [PATCH 29/32] add custom datasource if using existing pg db --- helm/pgwatch/templates/grafana-deployment.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/helm/pgwatch/templates/grafana-deployment.yaml b/helm/pgwatch/templates/grafana-deployment.yaml index b187a06..7cb8761 100644 --- a/helm/pgwatch/templates/grafana-deployment.yaml +++ b/helm/pgwatch/templates/grafana-deployment.yaml @@ -113,7 +113,11 @@ metadata: name: grafana-datasources namespace: {{ .Release.Namespace }} data: +{{- if eq .Values.pgwatch.postgres.create_metric_database "true" }} postgres_datasource.yml: "apiVersion: 1\n\ndatasources:\n- name: pg-metrics\n type: postgres\n url: postgres-svc:5432\n access: proxy\n password: pgwatchadmin\n user: pgwatch\n database: pgwatch_metrics\n basicAuth: false\n isDefault: true\n jsonData:\n sslmode: disable\n postgresVersion: 1700 \n version: 1\n editable: true\n" +{{- else }} + postgres_datasource.yml: "apiVersion: 1\n\ndatasources:\n- name: pg-metrics\n type: postgres\n url: {{ .Values.pgwatch.postgres.use_existing_database.endpoint }}:{{ .Values.pgwatch.postgres.use_existing_database.port }}\n access: proxy\n password: {{ .Values.pgwatch.postgres.use_existing_database.password }}\n user: {{ .Values.pgwatch.postgres.use_existing_database.username }}\n database: {{ .Values.pgwatch.postgres.use_existing_database.database }}\n basicAuth: false\n isDefault: true\n jsonData:\n sslmode: {{ .Values.pgwatch.postgres.use_existing_database.sslmode | default('disable') }}\n version: 1\n editable: true\n" +{{ end }} --- apiVersion: v1 kind: ConfigMap From 3c1dd78c60acb345ed900d22b4109b59c5068ec1 Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 17 Apr 2025 08:04:55 +0200 Subject: [PATCH 30/32] update datasource and basic values file --- .../pgwatch/templates/grafana-deployment.yaml | 49 +++++++++++++++---- helm/pgwatch/values.yaml | 5 +- 2 files changed, 42 insertions(+), 12 deletions(-) diff --git a/helm/pgwatch/templates/grafana-deployment.yaml b/helm/pgwatch/templates/grafana-deployment.yaml index 7cb8761..5f91801 100644 --- a/helm/pgwatch/templates/grafana-deployment.yaml +++ b/helm/pgwatch/templates/grafana-deployment.yaml @@ -31,8 +31,8 @@ spec: value: /var/lib/grafana/dashboards/1-global-db-overview.json # - name: GF_INSTALL_PLUGINS # value: marcusolsson-treemap-panel - {{- if eq .Values.pgwatch.postgres.create_metric_database "true" }} - - name: GF_DATABASE_TYPE + {{- if eq .Values.pgwatch.postgres.create_metric_database "true" }} + - name: GF_DATABASE_TYPE value: postgres - name: GF_DATABASE_HOST value: postgres-svc:5432 @@ -50,7 +50,7 @@ spec: secretKeyRef: name: pgwatch-postgresql-secret-pgwatch key: password - {{- else }} + {{- else }} - name: GF_DATABASE_TYPE value: postgres - name: GF_DATABASE_HOST @@ -63,7 +63,7 @@ spec: value: {{ .Values.pgwatch.postgres.use_existing_database.username }} - name: GF_DATABASE_PASSWORD value: {{ .Values.pgwatch.postgres.use_existing_database.password }} - {{ end }} + {{ end }} image: grafana/grafana:10.4.7 name: grafana ports: @@ -113,11 +113,42 @@ metadata: name: grafana-datasources namespace: {{ .Release.Namespace }} data: -{{- if eq .Values.pgwatch.postgres.create_metric_database "true" }} - postgres_datasource.yml: "apiVersion: 1\n\ndatasources:\n- name: pg-metrics\n type: postgres\n url: postgres-svc:5432\n access: proxy\n password: pgwatchadmin\n user: pgwatch\n database: pgwatch_metrics\n basicAuth: false\n isDefault: true\n jsonData:\n sslmode: disable\n postgresVersion: 1700 \n version: 1\n editable: true\n" -{{- else }} - postgres_datasource.yml: "apiVersion: 1\n\ndatasources:\n- name: pg-metrics\n type: postgres\n url: {{ .Values.pgwatch.postgres.use_existing_database.endpoint }}:{{ .Values.pgwatch.postgres.use_existing_database.port }}\n access: proxy\n password: {{ .Values.pgwatch.postgres.use_existing_database.password }}\n user: {{ .Values.pgwatch.postgres.use_existing_database.username }}\n database: {{ .Values.pgwatch.postgres.use_existing_database.database }}\n basicAuth: false\n isDefault: true\n jsonData:\n sslmode: {{ .Values.pgwatch.postgres.use_existing_database.sslmode | default('disable') }}\n version: 1\n editable: true\n" -{{ end }} + {{- if eq .Values.pgwatch.postgres.create_metric_database "true" }} + postgres_datasource.yml: | + apiVersion: 1 + datasources: + - name: pg-metrics + type: postgres + url: postgres-svc:5432 + access: proxy + password: pgwatchadmin + user: pgwatch + database: pgwatch_metrics + basicAuth: true + isDefault: true + jsonData: + sslmode: disable + postgresVersion: 1700 + version: 1 + editable: true + {{- else }} + postgres_datasource.yml: | + apiVersion: 1 + datasources: + - name: pg-metrics + type: postgres + url: {{ .Values.pgwatch.postgres.use_existing_database.endpoint }}:{{ .Values.pgwatch.postgres.use_existing_database.port }} + access: proxy + password: {{ .Values.pgwatch.postgres.use_existing_database.password }} + user: {{ .Values.pgwatch.postgres.use_existing_database.username }} + database: {{ .Values.pgwatch.postgres.use_existing_database.database }} + basicAuth: true + isDefault: true + jsonData: + sslmode: {{ .Values.pgwatch.postgres.use_existing_database.sslmode | default "disable" }} + version: 1 + editable: true + {{ end }} --- apiVersion: v1 kind: ConfigMap diff --git a/helm/pgwatch/values.yaml b/helm/pgwatch/values.yaml index 9517f9e..5bde5ac 100644 --- a/helm/pgwatch/values.yaml +++ b/helm/pgwatch/values.yaml @@ -8,7 +8,7 @@ pgwatch: # define is a database for the metrics needs to be created or if there is already an existing database create_metric_database: "true" #"false" new_pg_database: # Will be used for pgqwatch config only, if enable_pg_sink = "false" - image: "docker.io/schmaetz/postgres:bookworm-17-1" + image: "docker.io/schmaetz/postgres:bookworm-17.4-1" volume: size: '10Gi' storageClass: 'crc-csi-hostpath-provisioner' @@ -16,10 +16,9 @@ pgwatch: # endpoint: postgresql.local # port: '5432' # database: PGWATCH_DATABASE - # sslmode: disable + # sslmode: require # username: pgwatch_user # password: PASSWORD_FOR_PGWATCH_USER - # sslmode: require prometheus: enable_prom_sink: "true" new_prometheus: From 2b0c21fc959b706ff0549df1e66a37ab41ecc26b Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 17 Apr 2025 08:06:53 +0200 Subject: [PATCH 31/32] update helm repo --- docs/index.yaml | 13 +++++++++++-- docs/pgwatch-3.1.0.tgz | Bin 0 -> 94985 bytes helm/pgwatch/Chart.yaml | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 docs/pgwatch-3.1.0.tgz diff --git a/docs/index.yaml b/docs/index.yaml index a11d4cc..cd1d13b 100644 --- a/docs/index.yaml +++ b/docs/index.yaml @@ -3,11 +3,20 @@ entries: pgwatch: - apiVersion: v1 appVersion: "3.2" - created: "2025-04-15T16:21:29.294425244+02:00" + created: "2025-04-17T08:06:39.604311384+02:00" + description: A Helm chart for pgwatch monitoring tools + digest: 114f2f8ac8e1a8cceb5c5062a86e54f9b45c742bfa3b04e9701dd03bc22262b8 + name: pgwatch + urls: + - https://cybertec-postgresql.github.io/pgwatch-charts/pgwatch-3.1.0.tgz + version: 3.1.0 + - apiVersion: v1 + appVersion: "3.2" + created: "2025-04-17T08:06:39.601137999+02:00" description: A Helm chart for pgwatch monitoring tools digest: 69410525096ac64bac0f782205d92485e0d57cfc72f2e1d1684933baf3c21100 name: pgwatch urls: - https://cybertec-postgresql.github.io/pgwatch-charts/pgwatch-3.0.1.tgz version: 3.0.1 -generated: "2025-04-15T16:21:29.288927563+02:00" +generated: "2025-04-17T08:06:39.588535932+02:00" diff --git a/docs/pgwatch-3.1.0.tgz b/docs/pgwatch-3.1.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..3d32c6f6f1cccafe4095e6c6a436a8e8bd078a50 GIT binary patch literal 94985 zcmZs>V{oM5!nHfGZQHhO+qR8~ZQJI=w(W@~wkNhb$;s?@f8Tqm&goxW{pacG>ie#> z)^!m?L!$op_xhsxL1QGT%xEGh$1dy1#b(T+&SP&Mv26Yi(q2>Zz*a zz$a;DYxm=#=hb<$wUxo=^S4U%6seB#0*X$sU9P5qRb$)vWGnH+guLG6Mkx9G7B8_- z){iCSr-_<-_!GF^yyJzM@PKVKP+?N64r``o6l^nbqZ6g0h5s`?X8D!w@6N~Du< zT!jJ(`d{I~|J+XSZ)WWj5{`eew(95|@ZhM>xB`>&;|=aKoCl6jz`z0!lTJbM6mXBj z;jw`4ebfCU62hMscdzXidE0c%tM0d4IJCb^1y0}k#Oy;(ok9Ck9(T63FD1|YyPDpm z>%Pigq(=R}cz%qDh;$=_UFvXf+H(1AbY34M7}4BN;e=)QxoueaU@*yM-s1f*IAs-7 zH|hW>q4Mbckx|ea?LAO0f#=+W01iW98$lR_q-j^YLX3nUl>0C&1Ad^E&yuLZxZTc%`rUQ3nBVr zhbtvL?}#EQYVmQO*4s)^+va5(lQ5lpiNa$#J5}Nh-|wexItGD~Ed}$P4J|2MK5P!r z%W#jw4RvZ4*tRf}#gd1j%^}hBr>(`-#`8yZ1A&*^y_!wARhg87FUQ@D-yOyex6k4C zdL#F39-LQ+<8aG}cse0=UW+&Pz-BHiaKeow|W6&^eKy6FT$Vra|rbHKR;jhOP;Up#Te-RP92?s<-ZhO z8?g^6aThN>h4pIWei*+UQY95iksC5q-Azjp6JmreaXU|^lh?zod3t89+FZ4VZo)&eHTvn>bql-T-i16G@`+r4_Vi*_$^bpVnWL;QD0o- zX~rXOeJn{xt&qse@b_JajA7uyRBGxSv{Ko06A<|il`GODkIrxc{b`oofK9dGG9{yMTglL(8KMrU!uJ zOaJ(*BLd%y^8uGQZ9_tqDnHT5&CX|o9+X|rpOZ?7Idq4M{&!Jf{V(s^2F9!zO#^#4 zT#PoSyR&G`W$-n~7lUdjT~|LE6fk>D-f!wuLDxdc z4%{!qQ}%9hQIykw)M~lvSw{wGS>C^%Y zQE;X2E=Lie?BxH25^p_6%xUqj8`?nJ?{}k3sh&g}L)^cH&+mJ-h&!n_tx!QH4hw&F z>bT(Ve|HQtF+1w=|A+>W>Fwcd8*R}RQ2P)FG?c@cSK5&%l*{%N#XH|S+$H_!v&q`p z+PZA~+_}8Ca&7+iw@3aWpg9|>@W?DFzZb*GhaT^Ay@aK13iV0u=%*J)ZR0P^e}$HV zX?~acLJx0)$LFx~iPozfX#EVnWBzKQI=ihC9mR}5%#VCiG!b}*HEf4B8|4S&2Qj{B zNM$m)46i~oDZx)rL73z_WZv-GW3gt3p|X)v#rW=WN1}zc$uzwHTJ^7U7sMP9G=W8k5}T=|?Y2{YZW8(qMn<5NfV5ER;EcEeSZ? z>k1-RG0o1;Occ!bxe4GLY1I`X4xf~-S%D@6AmY{1QYZSjMk6AdkUC>Jg-Lq&lVHb? z>TILG5+Ui79>vzUg=k9RO{j-e;9tq+Qw}rmmk(aRUIZ0qugfAlTudW0!fUH;`2!66^;4iuXk4`+K zqpQty{ty}Cf3|7?BQYW9R6g#u-dJek5u>?!ZCfYqAlyrSxu=5{X;F+!kJkDW#e{cA zoEj_S1z$={i4j8NH1$A(L2RP0=3blPQ%ca-_dv=(&>cxTW+fdE$4Zup%}?-*77T zv9e_3AhV)gSJso$e>Y3R`$H`w0j}jD+ z;XE*$`w_~4i>A7=*6*}+cidse-p^{l(eLvS1|xp&Cgun4u{PP0fuTH2|GQ>N>@xdO znqSOGrAt3_nD;m$LKPyk&@o~S_9Ipo;?{C>>D8=7#sz6A(K~*BZlIBxuq2fLEHyO` z0&h!$x~LL%INL6w+j^wwhJZP+7_u$}K};{StuX7X&N>T1t(yDRyaFZPB8uCWgA9R$ zU+{N$Un5wQrk~}(&ELFex58CDe2oLpAQQw+-I&`dC1KW2w!ys~XZx|ebb}FiM^7(z zFCRBKyY-ILyHt`Ixh%WIwxz2u)5`2u$yCWrdTJ70cYPLp46 zg)0YK-7dOeLvo+z=wCiBr=S=7=`%Oz?0ymcTkYie4D#{}yMGasXmfJ(@ePuj5C{Zx z13l$X;RZnpNwJ}skd3iBF1Bo01jw&blZDyjR8ef+{!F%Fnf-G1OEt%Iw^LId8O|>Z zTa3DAB%WNL-udKmOJZSFO{W4U%Lf=6Y3#>Ha1J-Y9EPRa2ubu@Wtnl0~-hw_0`EX7I9Vp}jpN6amv z8RvzV5N|GP87*w^XbQGUrEq#E)qz<1WPx(Wl*NsGUaUSzHyLsFLA$^nwP*-&w11cj zuCQ$femC#q%C+34h}lYpe2_?MKpF1 zOJT~-^&s|L9(3YpyZ!z)@qRNA*!!$~e<@bcK(L3bh+VlnCjg2D_#FZkHQ#Ex2>Q8J z?+Mo_(uqx8{L&8$I1UQ24dCy4m=S&66At?kETs4RDbY=ZmRJ>yNP-p}SsgmEEBSJO6#g{>K?sS!RZj^^m9-RJ~*=T^1JxdHhFfy z@4DaG9B7VMd5Y&}Ej9XR>VM<7AMm<>l_=tw*^0Ym9Nt+9 zFv#ox2Vmdynf8BGLEmh!YL(uhg2+KkYcR<{@W#MX)&n&SebwC$`WMd20)Li5CKG*a zTIwp-c@*N>M9P){f^no4HU8iGv+8n~UkX#QDLVoa!iNK1)7?b~SFyt4h z)l`}ThYIBpJ#h43$F-kCN=F!OV?)$1|6B;*^iVZ%iXDm8#NC4THsC5b3q?ZWb_p2D z+gkad2ja^0TlvSEs|S5Qmxd;!=7%a{dp3rx=K#aI9z8Xi?_Vfq7pFZ|sIf;NN_I-LU z)O6qbVTQkkiJ#dq1N^@SKVFH>V} z++F9brJG$NmgVQTk zp${haeWvJWtz-%=P9rUY$vPvuH=p=s^-E=J!cpV>i@fPN{7Gau8vFqU;-O-0K~vx4 z)!Kh1=BOO+jsZRH`5J3E?)T)*6}85EK4{#v6=&_V?vNAQ7#r@2qb~G{qmPlJ&k(NN zAtx?7hgRRj+y0-A^2;813z+r#NR$H)3^?!o=Niyj1Ect2oiHlfED}hhA(vp+u$}^n zJ4$2M<@DK8ntAE+hmSKV-`K=2^+ME~j?^jTTVWd&ax!%1@yv68^VNVFlu@b6p{t}AoVuJa!gbM1O1! zor{fAF&0`VeWz447P`FwV3CWkBcJ(_!LE?$fU49Z9$rF{Em!X@SyRZ0Ht;~B>NpWA zKI3|{sb8Z#X1rRl^9~0oCY^=rT1dj5sx9zPi^kiBTdVQs3Y?kZcUHkTNy-SZ+)J70s!It5=2G08#VM|ez>P;vVm0mR= zENhEv3{(wX%OF+l;i6y_>j{Lk5~*#xQZ-)a?RqjteUm>(ZQ7BK3ly#cGuw8({U983 zk>|CeD|mGX=PDVY7E6v1wvc1#r#F zzkw8Sz&KK+qJcG(Vo{p3q*&u`!nPO26^J-pQNkq=Bf?2Q-bf5QDbs4W!h5@R+{iFq zqe4WstO868VOjZ_aDKwT;9!2jLP8!dzz8UU1(KX3oo>mXT!H)J8Ty82T}r@(OE0jYtLh}IXW_C=rZnhs40y0Kcf@VRsSoK% z-dJIj36gnTqBUvO+ws-enlm-lJCv*WEn38?xal7m z`zb=Aj!FbwZrB~?2=Ry`XpBHytQuv}4IRvqzUUlytVeLSsDs|i>a?v$cOo0f>VO`v z1*qeaIOaVgix+mAwV#zi<|P}GS_U5=3mC;OXYRvF*Uxo3uG?;%k2cm~lARcXLsWb| z@H`=L|?>Y^t zDvFcD;cen@L#w8gBRuMmNxG;w&4rR|KTO@ZnN*b9?4yR^c1E%R1Y%N0jQVNcqe1eQ z3RP7R;|OpJePW6fh4!sOlCw#clx9`j^wc!_3h*`wk0r~%GYk$ z_2l&Q0|!U4eZFGK{;lH9{jrY#E7p`+cgnB@H*U!gv>)Z0!DV8P!+T17`eq zHW-A~dF%2jj8!KwD@jVYrTAYdtU{88fx~wg04#=?<4ek)c-!8Biu?-;V5*|dB_J3> zaoDD&s<}UD6W6>+zZOsxkSPVA2r2P-J$jn#CKgv86Sp?dkmBHfwj%6Q^FlQ5YBbyogEs_t6ZI@RZx@iNu-SzJ(N^-t9iwR zQJN-V0Xnw>D1smO-+V$`D8nx5vjfRq4F4!=6pm7j_In_G^-Pjb*rvZ)N-oA2vDysq)x@ojszepbz!=_AM+`R(J8@_WzzWKP%9c#jMx{SOLMM;)?= z5Th%;0;6SBvcvM3Y2z@R{(NW7NtAjGsxDR~35rA3-Y@I2^M1#e#6gSS9L^ZlF`LJj zWMw23Hml@m4)-NChWajvJSzlGx;w(Ut&eYFSw=C`%jjq0Vv|G_){(Bi?yoA{A+vXK zGLO;NW$N1HS;VhYoH0!f2M0EqAIVngUXXrT=||m$urleh*W4B?oOvA=ZX2`=tvDa8 zWZ-21rMs-R9%+Swe^f#UH zGj)PRBHM8co}OyjMjzL(Z92j&csZpK(E+fLZP{bEIhHyz$}$=!wnvnjnddbw`L)xl z!)M;EG)v>_BYoOq43{V|wkvJ+mO8dDl-*Wg$UmoxOBK|Q8?GA8-wvdQ0c7AvOKxE# z%NH`sz*BrpTDyBNUoZTkttjX1xfDE^N?IOSkNSsVV%xl*Kg4rtf0gY{;?<2)DFpz=qlpOQhd9O>saAq#n%+4pCU(fTIX6nbv?e( z>ccLVdX4^j-rsDQR~lxa6<9Sp>;8-N{`~8e=}*}u`{gvl)~5YwBL*A|ghpn&GB;0F zIu>#rjsePoesH zQyIRN@jjeY_uc7ZmgwJ9Sxn>8AiluNy!o@u=l#81m*YigrHbRz5=d4cz z$6SYAU~h(m_5x+{rfCRZ)-mw9)e=#_0gf%g08RZvqvw^s)!OUc_wTXJs`ts7t$(h6 zlU_5$ua2kQlN*#&@yx#r)~&e?biY#{w<$uaZ?`pGY|J1ps_tbhhw4t!WuDO3GP9O@ zi=wp9A_mSj3~k1AY|8&Q3)u|o31AV_bmP>kbRnB?dnqi=OWm{L_1w9#ju?3VQSP~^ zGrHwf^H%3^bAqrTlU*VJE@W|`!1l~O^+Da`_$ZF>yLW90atLVm2TweIYDZ4Ejq2y65YYJiWMsI%`rm!Zz(%Ia_FYNbgGzTKSi0&QdC^9O;^TP4M0@!hwj09I7kZ;fLH!wj;5fBb_u}CIBTMlhw2ToJ}p)*xjFDY;$vVEO& zQ_t%e&2_QC2DlE8tj*@KtngV@!cd1QO*K8s%d%O|#bm96JU% zpYy!j@6PMpfJ@)#ZTLDkL3(d*J6sB_p)bIS|IiZTkf-kOQJsZj5at<4)ZXedu;`U1 z*v8+2?q!1GMi*lLJ8Sh%Q7ifTPWO(nhl$^`A%fJg;@?~bVjQ%Cv_wSIG5}e-bU_SD zaOfx47JaU_Rw=A?Y<;-j5K$}|)V&=3@YO+(9qP7es0JGa31%EpQ)9T$-89w{Fa}dN ztPF^8JACTn`sBW|zsv(%sO#_3lBjN^OEvDqiHGz^d149`UtvoG+b8rrtYH z=ZGV5c7E6Mhzc(%OOM2xF%8GNy}Pra@`EZ=#h$Jg z08+vwS4pUDn*~n_*RXL9XFCRnA@u**ju^l6=y``hYVTCV+K~2%dR5%;dz^w83cWoQ zs=1dHF7a~;3p40g!6u{ZtC>2{cPPkYmX$V*lnq61)p9OHcbe`t<1!4|97Lz%ftnf+ zHk?_+EE~nL@vxZWbfLXS*tg=H#C@!jOc+S@HGOgHPR>Ra;j#vdZWm)co&jKy>4f&Wv%=zPT4XRz>XK* z4ZWoaY#I6x#FM)U?xt6QrgpM*KerpFIJ#803%VY#$LSwSp;bLvn4cB9inE)Iykhn0 zjNH4!I?mZzU%w<`dz6MQ;;>VVVR%}qVH9=}Tnr8P*2d4XiI4i6XBf#xV7T`P%>`li zF}sp%s6BbuT8|Kt{?z6~P70ScjvkR& zNV3$k?tjs??%X;m^kB)?h3fvz)BU9zeH8}~a<00cOxw_^ZgYHA-pGWv$qGNR7 zYUScWF3b7e)`u{VLw?sB>=U&X!cpLgzXQP+4jW)62Iz%tIh0XlyJNG(yJeH$+(s%) z8YAoXltqWtd=>Dog@4&bX`ZSwBNFk8U@=cYm~n@kO2kd^BZdO&24RXfL0yZm3W5qX zQPs)GS~EJQV=$X1<+Y90 zwHdA}hpS#P&)Rbz-S&ZOL{GROt5MEkS@)CtHowm!xfopP+?8g;z3gljWRmqq6y>~X zkc$@MtGoY5sIy)-Z;y-K%SZ3#`Ym-QALp`Or<@d(zDr9!Ywy0tbSDEYivNB1Aed`f zx?bp6&TDyhRKI}7$dAeX0vcqb0Ib~4qP>-Eu;pbi)Rd^slhwRXqQ<0}tUeVKyw(Hf zBz5SXu?^pW_Y|TKsE>)X7u^!Dao0uE6d;utYr;_*`6D%k4GRU)$v}0kUE< z;kP}{LWPk&=Z7MW3MD@qP+IGMRD#{bOh(M1HjW%Ieq?kiU9gtFgO32m3Z z)gnf#YK7#Fl_2D=TDAqzJpmsYjTdl*vBPNZldwD(gR}e^9`p`}4SXez9$rhej0vWK z(2fjebi^FF+AVj7yQuPAq&zQXW=sJxO58QUMI1fgnhC!S8H;B9;Gnv00gpJB{+gk_ zKG))*`^Z;+n-vD5Hx1XsV^?&L$f~P8e3WyLXbG8c(I@V~Yc;L_nX4by(Ybl(-savhGb(pwZLjQ-Jfb)wR~t|Rx`J^%19cJ+THc8i%H(V&g5-8cOf6ffJ?OhZqfju&iBmdE=E$;BQ-hFC zufR}yJQDf6G{F~gny%6ZiH0k#p^6|iMf4@P_d=FikntWOPTndi`_JfE{VDFKLQ61> zKj=$iSb4yk7fVu(M<2}c(sm%yh4INx^pXZmbszbv+4{l84Q*B2K{3PEBYJh(K>uc7 zbtUDZ@cp~;)c_ku19xG8JtY&iGMs!X3grj`mP&>sP(@;aH362&!^FZe+OKyR<&^bY zf2GdqGAZD^Snr#GoiA>u7nw`5j&so`c=)tIZMLm?#2o4V`igH;A*p2jT=<_u^1$p_ zzBT4G;a}rPF~frabrJ*z+!u4(pS6h~-^T1wFv;9r}X zIDwE>^nD+wzw_OTO_L4Di%pASH}X>{)>%KbNv|I)k*Yn*AJGR|3ZAh%%SC0awx#an z|2=fknkRPMyISO(BgX zrOyH zU%8cDq4g$~I*bfEokTc79rJ~KM)@K-q@9cA3OEgX!Qlo(8l+EVPxw2Q2D)A6-ui?6 zp4uQTTlr4F2TQZLw=YTQdrH)v-n$hda== zv{gfdXL_X1sd6 zs;hh|##k>atQ6$D`K!2~A%?1VOg}J#pjm0Beczv97b`O(fGL<$hmN$f7-CDn z)x$--mnO%jeaurQvOzmaZ&2BfTf0=0H8j(UIMUjTlG6XJXUL#&>*X?n@-ww3zL-r4n!a4Hln5F zOok;cYch9pasOkY+{RYiYo%S*<$*w)JVUK~e6~gYj=z0xALmE0r|Z*Ev*+u^J9-b^ zH>KBodRpg{Zw&i-t@{eA$^q&EJi>m{WnWPy+;{zealSf396U^y8(S9_gECq_>yA3F z*?XoIHR5?@B4*>f;RRWg)kU+$6oUvzIK@FfHOx{~PG+AmXQV9oR$FnLKiEbHA7LWc zE-72~{~O40VBw~Sry#>jhN}~mCUT4z9Fbhc>sgt@&K0#hE2_z#ISRcgL+yXRse#H? zV=Z0qB|L=x#i&f+ZC?-})e}%G&=2_OX(9b1o!1Rz#Vgz%&*&5p=bmdP zh4d5@$o4<69-wSTUE85Ml~a_$MtNWp^q96HMuhLkSJ`QUNrWFI&v;1sFbdTha&^Sl zaAvu$8J+l^`yJHVSVQYms~3+*o|*Hv=LG}?wG+DF)Ln#im~d2<(kTGU`^sJa z8gMD>LT0KMLV`1?7$f>`($+|eKw5upDVn2jF2^!*Q<{c9r+E3(+>TQT(eY(3Z+h zEM$!3-j_M*aQ7mI0e3IiM(H|CXTlb(QnO_zPLa8fLhvq(^nX))KdS@?(mso7i&<76 z{n>vqJlHV9JnR1}!_WH2dXOxc`ZdN=|4Z?MtN&kVrs|k>fb;*P`065o5|~MG*FVon zmb0Bj^_PrdHZep4vN_b~KR;hUkEnU3Cz?Yu_G!i3SiBoARQ~08)yzKeWk`d6`WD%M z)$KNq@bV1f14$8Tl2c9!QZ-cV8Og1O>rZ!_`Yx2O6|0_#7)e~4cQugdt8DNKSC8fu zwp(`|I7S0EoPL+f=I!ov>Nu&_UxxGtb1b4*7hdo{dDXPA)?qC0Fd|H^l|4xWt*W$i zF+@>e?UU)=f9WsHnn-M7^F$T|xqf-X5zp>Dpx(#OVBm#art4n)j?K27dGAJ;N(6lf|;aDoPkglXKqahwmDbMnZJ#Lf{0 zyb`j;YKt1B0hi&OlxBuUz*>xV>-_F^-}3l-1Ig}jhkK-x*TTArp?L6*Anj0D=8>q#z^aP1Hy3j3hEbpcgYvfIg|OUp_M4tIsl!D+)BME2edWO~J+9hKEUBjkt>*#aR7FZsV7%hB0C zkA;n*If?2AMqGz)gsU=p)l6MGCn5cHj)8jDpRx&SReVx`ZC80vyscULe&!LMu@|Az zuyTQ9IIR$2{Yb??ZiH%0H|^ib6iy^X+U3_}hfKvwh4sVSoqf*8M;CYP*c3NaB^buhHdD{IO{B+eD~DSnC-J=7m>nyJ&wcs&Iq0?o zW;**r3KY;)oi<5$DNwqO&R(8Aa=-a~0FG>5Gq;F~<7U_9*{=-!rgpoAZ9%m{e76?A z_L}#3W6&)K%K6p)q`5?05^@ax>VJY2^e0RqGBzrJWs-7}KbOU0c4dlJgvVfUEP^LA zo-PA<@o8h+UUv98!64wr*9R~Kl3dA_M4T|(?mqx7h8l$N4_Vi$rNbp}sWRoMv_5l- z%CHp&rGccg*OW3G*|0Ux`?jch>7X^PmiMLMs28VgR^q@@3xv$H8Q`edegRqq$~!P7#G=K7UEWYqpP2AFk2YG7$my9K}=%oKxLX0(ZB{KS2x&<%}HG=)lk zJNfSzE7vqcMA(Vf+PZ78iZd`?IGsK{{s?v8oUpr{lwe4Q_y-~;blL_ZV>GCs!BCsw zNKG9Mj14s;9Ttp&N8*$!Ke?>{O8x2!Gz*cOp?~&Mkr(E#kHg>&QS>A2&iy339PPwL z1hMw~PK4U^tY`_W@5&{~dD-Ejs9Fea!{Hy%*S~;62&nuqB~|Rw!|dUp;5H|T$ZnBO z3(YUt1>$ZAD^h`mFzkss-lX)a5L8}(boMuO{Adxs-r@B+>FNl!LTY$M8u?e_3Rh`{ z1y+Yb=Kmva5RN@yF9@)e__Gj{jQw^@(Y>d856#27{K>YiVO~4mEB?;#mVubrpq_$L?`&r8;*jL%#3tq1mTv_|MkRv@wKp z0XfHx*bIUJusI9MqTOUU1ZOCYQ81Bk4?5bxG9@4A;V=%`2Z&&blJYynbL5aheVnzL zh!t}5Ph)eLu12HWlz{9UDeq7ax1OMczY?|X9sUB#_W^IX<`0J@a;kcVzNq7yFT#Sx5N<5pVrcx95-aS-fGSz`1w&Er3{i{YL=F z2{;BlG0ln zXEYxtYl7ZBWc?Z$UH<(1F06McwSd2o--VSjqI4o@dkqmFwftRKyH6;P zF`^4-JwW%HtY_H!-Rv1{0WTtLbM8W*kC1kNcA(3AO^?}Bx#MVBSp+f&{4i)gXGM$N7+QHCj$eO$J7CeK2$8iiH6CL=7Z=`+**I3rh0&>S%MlyZTcF zga|x{Bt79|maLqLrxjM`QRnW}`=SoTGU40d)-r19pCkDpm8%uS)YMHvENJ^Do|(`y zDwcUg<0`8k_}>aSpUv=ChRoS5N(_l487wbw$VBGC_K(U3li)ruc^}RzD@8oD(T%DXUXo{C)`{Fc-+wo z(uYAbTftnjC>$M;z-XtBcG0p`UKts5<_f*JoXoHi8{I8$^Kjxh3At+=8zFHeL&ks$ z3nIS~ahMqsVuW}h=O1*r1C`R16Ov64A;hMOV^>*H)XAOSu{YAhZo9=lg^RETMl;^fVG~O%*YILmonh`!&GadPs zfI`opu1TW^b`cvq+C1LB{x~tmEPeJ(xV{&DAQ~0UzYos87iyw7j+jA5GdpT6dNqv8=j&`gBUV&iVtM8mYuP_1AQSVr}_5Gcz* zEdDtI&oc)77l3v$%pan=_iIaBF&{UhkBM;T#Ej06AjADGl=(Beh&VeF8Gr^xfe4*n zfqthrW5#-~M59L_z8_~<+%B;XYzzKSlKIhB+gI2HfFQT-uy?R_2Qa00ggdXmcP(7{ zj9hpCYO%kQ3(Ce^$YI-=SfuQim3gJ&RUna^n~b-Xp&^+(-~a~=MIXpeX->wOa@$ps zp9~zGOQJABzx!G7zj{n~^ndl3gMWI=AIU43+d|UuYXbgUpHI9|M-Qm%QV3Y%dyoLq zYh4OHN;UDr9Qf0L4deJjRGCDBy}mlI9%LDtStVz*EY%5P8|6|&H#(`*DoRHua@aj~ z42>+^3JQ&o^`AKl1YguPX}+k`^VJ@ZU$DtUI8Xt~P(yAL=mQqCAv}&De)j78y$Fey z!%@oRz++W>tVktOb?`B2y)b9^g4fGu?<8ZbOdT;C_-Y5DVg^ftj;~6n7zhmoGbQAN zD_BV)Ua1eYQHz3YB5adVC;8l&;a6wExXLKXg2K!V`z-T}YSaysWy*B8$x2J0`#>gR z=4g|pah(}xIqyOMOOw*YS7ffKyV@G31ShXaEW{h?n@rsKs8g{7WSAvWB64>4hE*Vl zS_Y_5%_XVVd{4Xej<8kacYm?Xtzw>Vb*v<+2bnZ)W04?bwG4e3wNe0R-=82h&&cNux6kF;XcKZ}$2S`%96P6Vc_?6Lch4gP)v?5;k@;@?- z)b0PsG%_RqBhw`9nk42BxM1u_ z(viSAN>Qi{nv%?EkD;rpBg#bV?r5QsNYSCBBu~WnGnAg$j*y|jZKskEKE;BLyi$FU z3heH$$7p6sw(^)r}sxopfsKvFwdn81qzZj(QTwppwL~wUV#5 zt=PlL0@aT2rw`|)LndPShs4L5X(>~EDG$qxqJhK3TVx5sm7W5P_Qq&^m6%3Lw&X0LfGhhM6bm6mw-%G$oqVNc>m=~@zT7j9~CY9+2o@=8sz z#6;Jnjj4!9nbp$WjZTPJf~HG-pJ!dn>1-cBt0fy0sHGkg%g^0h1Wj_I&RN;3C%Ypr zvf;t5nga-8UocI-OdF#chgpvdlJO6aJezh2ZMK%N7tKziUd%g~7x(1)f&#=4NJ&eh~xHiNZo6% z00WiUSktsrBX0;xRyLy-)2M<~3i2As=mlX_CZ*w<&M3-D_P4dO^KC_zdo?o0l{qfQ z3}#|g@w+PI*~xfN2lC}7e7gA&)e!jkxBtj}4!VK4VXz;$_fsOajs=_l$p+`IUOf6= zvtV`>ZK^utII1aMDlAGR;5ZqjK5QD1K18CnbIY&SyjxdiKso^ZWV^3acdB3qVW5eXOw72nJ{O*=UZt* z#0c!zaLqf#RI7gblNik^th|JLsyGxEk##V*VSoF-yZzczY~p*0xvvQ+eIq#@(cK6X zR*D)Hd_mJFyY~XG=UVsmKZgM8u?cOFXK1w0MMvmB3y)^!L2VObO4}FZ!4vSq;*5lN zS@LA7b28rJu6m!LY1NAY!FyET{u(OvnU%oHwEXp$O}nE-@)Ya!Sd}3ppUBtoZY|Qn zuB7`0=T2f^a+5Yc+cmEtQsK8`U8z}3%bc=Ij}MFfR+*q29_4(&3+V@k3MECHONR!Yj#fubgGG z(X<*2kq?VZFr-%^1bI%(9So+v`suEqK z1Tuv3*7z(Amy~ft8O8Txam>7p#N3iS+PZx{n+J zAZ`dcc|;GP``lr14m*=1iZSAnn8zAMK<(7k!#Sm~&a!y^ZE3;I30nxcx-AiJOZ)SS zpTF{&Nnj4y-p(p|qLH^4Y3{O~LQtO}W<0BByMe#X-#QBP*cJcfk{6acRA2&MZF4;L zlPpIP*VXQD)}XBC>E?vt9RHUt2(qw%!mHf?fXdM_k52WPlVK6ym4_$MNeBS$SvF@M z_5l%U8#2xQHGN&=yUr%HINQh(X^Y$CBsYvdqM1y^yQ4kx!vBgs%g;br^FI8X#JP<4 zxVy3SU0Q4r$4tK(Du z&pWS9p@s1B#d!Yb7QH;>`NZxJZ6AUOqP%>i0Wh6fMzqKl(u7%T9#mAxuXjNg?a`U) zfG;o{ddc9ZwtNrZXbzmd=}OYKX7mR&QN8Zr&G}A_T0BnM%dcauSXpUNZ~hy&cxYy+ z2k|8Ierrbugo9)V1@VQj#tVi`x)i7C#JB-#k^a)5zeC<=YO3#($G*^H1-A*AN)fir zLGY4+%_zK_d?Il&1i>vGypr#i=iVk}zIs~=;0P{wZvQY;_#?D01%Nc&-%_RlKM>7= z*xdIou_E~N-}kQbJC@eFaGe9(kd0>cVI^cAt);g}$XUo%AKNVx_E&!YwGg>ijx%E! z9wO|gEb-~1*mbT)9`)`J2}51)H%oe1dYS+a4jk$ynhuX`AlQE3y|AmU$J;wt3*yM4 zo3jlxhe>l&vP*ue^s3Xpw93 z1&qy~jx3!z-Cx|l&(1t!l>4*B|L)CUXYlvlDQX+C0ZEwJ*2fu>TD}qZdfahj8+63S zaC$w*${`GUi!kPV>iLoAtv|rd&3GITi}20mkp_FR&<3eyy`QZ^#bD>$O$&-TJ5nYa zk?u(ylc2m#UH-9*pe<^?#LV;kuEFH}Q_$(j<4TyFjzfNyUu=W;Z*bwP;(Px?G@#cg zyClOM+3x56>GCQBZ8k6JTPlT%;YK>?>2XmXfB}V^Gw#F>XAPbNigkUxa5-p zk}1j(B0$19sT_zLq#|1yc*ButCPmudv3Q0@eeT%EggAx#u=Nj_%gOsxvz6bFDGiOZ zf+roihB?k0>h{{O-buNjs;j=O8;y;s|m4eb5Gzai3f3pf-n$eroW!<7dlf5?qt z;YL)8sJv8NO=6u2U-+rC8ejI|lYvR1>h^7qa(}UI9Yzu`_1TX_X6!TpBxM119htB4nkel3+o6ne^9Dtt~SxuxnTC>NgGJ~6$YgS2VZ>R^~EINFw# zT52Xyv1~LBUuv+H1X_MARWe83@w5gb1+w_3QYJkw$3M%O`0ZQItwwr$(C?POxxoH&_uY}>}|-+SIu_f*}g`%mw# z>Z)gV?|!hqd#&}^6S|7GpFKNw&*bTSZ5MabS1Cv*^gU4jqU(UpBAeI0-Qt&{|9?e` zmBvn9U;UZP;4|Ik?p`YNZyEr>ewPYlk0VMG4b<(}^yS_Pw$b9jHlitepz$V5P)~y9 zLWzxaA_uB*TQ~G?H&tWpFCwYSD1fwP#qZ_z?gIo%s5J49{E;Uff~!i7q0^QG_|0Va zRO|W8L;*<#l%B>zH_Y=IydnW#M-;;c5r0|uKl*Xe@^KK;KIc1*1h}vlrYoOjY!~bz zY-KOQ6zg-3@8h;{j83Z+kAzDU?>G4*thu;QB*W$vUdpWCHvj7&($fw?|9{knIyPA= z1N&OB|8WrQL1g{^1a_;#xLAA7|u47Vu8Yg;;C2< z!x4NP^BMCZhH-#ylvXI48J>55q82SA*yLZ-_KvXAD~5ojI9ly zeLHidy)ZIY_T=h@D5Z(vRQ0Gv0XkUnag+rp;3J?vX8z&}n14D};}ZWA*&Fe!|YYVzZ7 z4p9!f&0a009ikm2jK@<-IAZDxKlcy~gC8U1fQtU!0K+rnTs$F2xU|Hl|c_(8e}&zdjowUiJ(z62N$3lZcm0@ZfP-Xb*GkY=ZHX2I% zst-c^HrxJ%Wuzle}tA=z3o8v|iA+cCULBMIrMZ^e_3k~~YKVt%sYExqL*u*d$ZR`zwS^Mt?v#|M}!TbAfu`XthCH=^B4Uzwx_ScRqLI0I_%EJ3;4xO z{V!9MJxG$`uEawf@|!w^n8Co`58e$;#fG#jTfk>(Lk_BNzwm}YJ}wKr1fPhFS8%?b zG5Ad0kc~vcS(L~>8x+;XTC@OlfEgu_@NR^+=Y<0|)`GQYew=we!XwSNFu-I6z{N^5 zVPqkgWLgRuN%8>+l7s?XYfKC(Nf0Wr$5vjCn*(2gXO7D^2LD-9{^d6u@Pc9lj|Ro7 zema?>uu^(&?$d7L=BE=RZ8!ORVm3G7e(vIAsV**(v(&pF;gkN&Bi_mh2}~%Gml-QJ z(Qd2adE9U1l@Y=$PI&s{sH^K(dH+$iQ~spyImLY2&%$OyIe2#u_vXo^jvft`=h<}U z;h8Mh$9Jyt=7e{fpdC#O%K$Y(2rqKQLwKx^=1INM`d!^rZrA^$_qwUzim{j~MN(IX zA}21Ivr?4{)JO236qKdXS(_)XvRNDV4%)h!)&n+kU;5ic z&(2{T-T$wY2Xu>duVopM6NNcHXZnBKjuB4B6J}FT+&uSPdvEcpUKlKrD+DPg+g5v7 z3bJ*0DHZen(_tmkGK(_kettE3<7!|6{W)J{ zn<05mhCWwf+Xl~0^}S68m(2v>`99R=#RuPYrT_22S@x3qhV?w8jMinm-cNX}m(WbL zi~z(nlv*3vDcxU&6FlBg-D$2p6T4eGVs8ae=miRWmch;x$G!BRmI%Ce*8QQm| z+LK;|*J-$goi*8M+U?icy`ePvy}8v;Ityu52?a;}a7F-)1vijEd;{+zqm{XtI(25!9JAo?>2r8tFitv)~3 zcTnnm(gD-RyX9_ykuMLgaPr@`Ktr#x4mtyy;Hoa$lGwCtRKz}-qB$PIyPqg69zV&q zwelv;at53yOvE?&wVVL1oj(DBPPhS4Z>(twK8f(_a?zH?9gh>qNu=TnP7wm!tPyzU z2r=#+rb}TDynt}4iinqnh9^v2!07fAO3GOFxsxb(g@A8qWdIUq^@gDpf;1m>O+L+voa>WPIS zAA!k~&kaz#au->{ypRzT)B2Ah`^`Wl_!nYQuW3RG-k%TjE`zmh@(xH7&?1i$dPvZp zs*WjwfpnbRAyyWCZi6kDw7s%zNwWX$yGh6p`?RKFsyfw{C=P2d9>u86CLj*syi2jv zRa6CJ?a&b&KO7w5&rjNry0$lhP~E`sqG&jVDW-{B?C^nWzqwjq+$J!nBfH7g=hFLq zRA+B_%gZe!s`7V~V$$mm=ze+PtKUbchsKCFW8;RCg}GtBs-KlHy+@yFDmKVi8jAdK zV`P;od*#E=*E}-Vb$gwE(-W)ZM8CCWp3i^GYNUoK=+Dg1?_S*8QO{S2MQSxb)o1`% zeW;>bw}IIO8}N}7xE?v{Wd-vIV>b%x<)ksE8ZGO8TGh+S?P@hGRyC^UoF}lq&*$PJ zdMw?zZ&Z$clvCuP(Pq7RoYb1Xh8Q@~%l$MUk8(F?&=YOIe+Quil9g`il_xc_8T_bq z2qNUPf5{Xq6$U66usBmEgrwGlC`0N|6R94c)-;RAYuX7C8*&lJtFi7*%tL)w8w`^c z#|?s8WroUoX_9o=3=BbL*Exob;yfby$>$7Uv7@Ig!1v?mjQjEiv1i{6AY_9OhIh+h56w^L2%XwRmfzDg{l{U z&W>~cVAQWL>D2eSv0>5oslC}MX5Tpb&xKu3wb7^QP$?$^VmdmaKAg^;b2B9Pm@le& zr5sYy6Q_pP*E^Ine3>QZY0Ib-E(IMbMJT7kJl}Sc*C+H4DH-RQHkm~UP~{L5!fJRY zrQbIL7dFfj1TW;=g9j02++7e-A~;k8SE7U(E(oWflMM)>=BH}X$W^1+bJv=s|FYnF z$9cRMoH~NJgFIQQd|Yfnor~bwbA8UXCT;Ms3S13jzxc|q+%rZ`dXY*vUKmeJQ&_A8 z$<`tT;J!lmo+_yagr*7w0HvZ4c@{Z*|C*)YZCXWWe|BTCn&5Qs^s-?6EtN37c9V$t z)f5M6Wmd@Ds+JqzmjeuTX1fqO}o7D?^YTc zQ*LoK|8XS#A7B=EMIbG4ps6XW0*4NZqA6Q}yK%NHDIUuv%}pd@oPB`m`$ZR(6hm^~KNj2^x;}!SuYLjj{u( z!ZRL6$1jzR6>?7h?QPkMs+}y?t>KzxIo0r;SsZFf7p0y%y;RWvwx95PQ_OOs9>Zs& z!dEzJdbD)hqR`wMOgYODd*emIYRrH>7X&7t-cGx1 zZM@LSj^yXqO*}W3#Wc@pLGEfH0UYh@)-L>e)pgeOF@K;ItCLH>G0Wfgz2W2U_$G0G za`D&l{;utr&Wop;9i-#pvc)mS1o8V`{~J*q=$CXvP4CMafHkU!rwlXRhDYf*#;_>{ zH%3DAFDjy=`KL2~bw$Z)o0=-+p5>o=xxm4&J0d>i9J~7aW=uh0_dB!wx5%XM>1Xj}6s!iC7&|l~QTa2c2>aj2Nak`l&Z6JQZ_d z()LuE6q${p-!gH09WJSz%#J?fF>@QC3f6B|!wWP7#+W*tl?Q?Md37j*hc&;~Vq@zf zgDf&)fkkC;QkRf@&W_B9-WFQ58IswVjA-UHM)Yj7{PT6+-Q#scrnbgwum&UUFd>I~ zk1!v?5yk(lA@_f9Ypy#{v3A?d#>GwHq7q*GV*4*=vP#Yo|8Vs`4mAwt-y2V{W@uaC76C9pW-56p88EKWWR%LVe_$MG z^YNTzr|+>8+y1{)CzY__Oc%cYYfzJgYF?$TN5y~iMPF;8hlp(8jeBNr90$hu>0Y*v z3A36Z%iTd+u|s2t&G>3F9+RLYIYgIYm&E3vOw_=q!u{s$`YHOwImx&h5Gf>H;Y*=f z@ykwBjU^e;DyrJqaE5@OryQ!ur!EZ@YB5)QClzt9|EkS26HUbb>v1dP{*{qPYO;wi zavCfBjH;x}S4h%&){e3NewEF!Jd?-U$&y4l`usG5v?eZ2Czv^^XPzS|0Y@p9bRUgO zh|KU6Ku+{`5EM{kfX9un2&I#gm+#BTA3`32y(*XA{>&Vk`vVzctslF{BBHvgCH!ik zmX)W`ZlCKHVoLDII3eH?mx{}Il#P&^Zdn3-%YexME%PvPr=S`Hnt4|*9Fe<4%*k`8 z{Onj8FU)EcjUmeJbi2M;@>EYAv253y${OuHLMXclV zMQeK{(1>O9w{K*Z51JhoN55zre#(3AG~{J}n$BWy6q$clhALR}CrQf-+oMwc%WqHk zp8~^ucIX7IKVC7mU73%4J=um?omRV6)23BWp9T-j7=~*VZ<3wk!ckwqp}Qs_D>P(T zmOQi^>5FU(%OPQ%%V$wR4vIyFY+PdD)U$8e!@B<*WWe%%C$xJ&(#KjM!B&Wb(I25s z?h30=SHaeqvIz}m$jRM|Onj|-Argb4pm8cX_mbXKl94-@2}kB?KK}TRiqxk}|H30C zrw(aP9&lUe!4g>A^4WmhJ<&zpTRChh?^Pf|4C4#00!N(ROl%GM0oFy36IMvbzJs-L z<#EFUC$-Dn;ILV`LI%I~`<6;2Rz73f>C(e___qIvv z=bYbo5#VKY@h^7n@_ zKSoFoJN@_9t9t=w_lO}wgZD||`1`Z!oVFY|o2@lV` z$ihT)nw)!oa%5J_fg}P^lQm;hG}!88`chVFrfmHO{SRhT9R8yxtY+Pc@v(tz3de8oXJa(I_5s$wW1->L1z* z2#k2!l>E7fT#u1-{V9Kaa1gpYZ@-<`KWsolT7ob}LhtE?3wJC>ZjV%gi~VnK;n{7G zM!Y~#J(=LSRER=`!Gz~23K3bH2y5&F0`EJ0Jmwf=Z3-6Bx25Kf8qDIVjb%EJVmtTa zu?pNB4DAZgtr9x`hypmTuZ-&&jP2C-neHJ2Z z<_QaZpx!yY9&?UwmRY|?=iyp}V!ba_$-rt1kdr#Y?n zQq#ErkO8c>-}_HeElce^*sf#_T^%SZew2SOh?G*R!aZ@DuvBU`Zk=*t?-vChtZS2> z;hc}x#tGn}k<-vf3o-HHQE;4g2=?j{hXTDLvM0#4Hs#B0-MDkni>+OH8C_~SySqSK zdW+||EL>{mJH(H1bFqF{xbQQ@yNVwtk^B}qscdn%l))He)gpC_9WGx|CcQOUN)_De z%9Rh^PlwKM^A~leL+sV3e9>cCzluLW+AA;ZCxqcafL6gKm1wtqf(_xM4+%Qg=2$su zUo$7gGo!em8&{Zg=RyKd`dDJ6VSx;_2!j2kiNmbWGi%=C5h@Q zK_ND?&CydV2x`C03%-L&zjABVlUB7FD4=I&+ALQqb}4lnvxEB7f%eLK$rzsl$7!V# zI8OB~t1G?_IL}I8?#d~}LQm2`7pk|iw(?-B>7?jJKjVsk(GPSy>b&;JKl9lHBo-02 z*qroG^O=K)quDd@0{7$ls78C9s)KFNK#ma;$=FN<|#|_{k-~P$f4{=azKUuyXhWKg1Pk#mT(ZUZ3=+^M82Wby`*JjHIGs4;MUQKDd7V5yRA?-u4HuYisM3ux@r|nA+Um#51Gb}I z?xorpABzVy7;>}LPCmG~Le4<*CH0YZNZ4rn5*+)#S()6lSiI%`xMQ|-MJ4vOYpGF+qm0;VEl(-s-{)>8|67Kt zph>3ak(&f*CZDha|7fA9YpRm`V^;!o$^XHn9onbqOxo)HA5z+u4>f*CF_5d$Nr=>_ z)K0Fs=gL?jS^gAl;6*Dyl65x!Tfs#o(8aQ|}gdbWL-1_gVHE{Eo-m zr9}dBd;S64CDzaC*@C*@Vfb-ypHyz@{5ZZK4tBk;1P@8fILYgh3E_n zS~C@5dbf)n&jjC5ZcE?i%Lz%sKR~lx)~I$D%`Mc?8T{0hq6TX0aMUFDZJ+kwnUP=Nw%ydq+| z0fyW(SSqUdF#UcTSAv_Z*Wg;q@mOE4Afv{pURHNlcmqnk8U&%+P>4e3^LCF2TZ)>lz|xUQ^@xv5Yy`bN>Ljs!ne=fceBjtqDp$1f)|Kj8l% z?KPkVjpJs4iBa4dSi@-&M+6=`cE*56vdsM+C8>;=KGhjbt4EUJ)hR13fQ|c*K*QUn z9x!Al7`D&Rw~{86FQ$rxR>JKgzan>H2M3CJK-!&yErduzVuXK4m2e$mFNZNnUW?0k zf1SBN7~MT7*a z#qpcuuD``*1~Pn>!HA&7gu`!GB+*@K!Q(DzV`4xlnVt{{hBT_QOVAztxXBN(PAU@1 zXg;Ooq`KG=C24>JEvLs%4oNxgh(kL1cyxv&5cT-F$a4=QZM20zhMKL~W6H$tGQ3{X z$z36Kj6SOwV!0--oS~|Xe;Fq?jV9u|&>0c%CA6z}5&5tEqQqhRZsRC7_^l*ims)kW zS;QGb<)dKOL6@nW-Hs*tNt9}e&LF8E)1U{Bb9jPOYsqi(E03&C`tHc-ahFEn^*Vnu zCjf^*1b#Z;!&;|(Sb;h%&wrcmaCI04-w#`E2i%xAA!@&fw~nI^{+@0_?CBLjPvPo( z1{pq^U}enYH&jn*#2NqOj3T-6#V@g4MmGkXwDw)d0UsrXwHV)J56K}3kAv?T)ZCNPfsac?oLnc(-kPnDIEM9 zy?ngxb{D-ttM9L``SWvqzkr;zXWpgJ961!PU?A!!C76F(O+-*PQ^^HwhMoKbazeRlEqM;0 z$cf>=TIrlat;R^vJLp?^3|!itI`iY(O`n#*;_lge@`pIRabA3%xo*2)%spHcC<%rz z(7v&^ZdgGg_ibP#I8|-lFXhHHzM|lBZ>q>T z&^z{L%MzJ(s090NI3mAbTayIP5Ttzr`UFG!1Ht|esPC<2Dc$1_ke&u+zawO%0C)ow zae{S-bPI)^nt^Vpd)%pV21`bU&t`=dvZA+^$4d5M2{soV8?6X~jF@_8%wM&=lltJx%Q1fBR<+nOTNFFSuF2C}EmaUZFiJl8qck&&Qk<3sM* zq{*(Wt70I>teUxRz8iJBY69+CwaRT%KIx!L*yC*(0+i>!)>utJO(%`&)@=v2jDCv= zl>uhfhk3F}D~zbDa@%xw6Y#0it`Tz#hebxc8jj#jL3|kVsOvXri12=g9of@V=ja9+ zynDWe?X#=|#ruGfaVvZf1lk0|j#tFlqO7iyct_gLbr0I~0J?=Y>#g;K?%}kgb|5KX zf4^Gos}D>I9BedTlw_9VM{8&FfuL70Je6~(=wBQ+)j~+?4HtGu|Ijsc`D1?zAFqx; zm@l!vmi1TuI$I=9H!%91v><+~x~G(E7i z3s=UPwx+>lr2fCAd32!;P6q3(oT+(qu4Q@&n>*jtmg#{_`=MZMNe|p(8SAeM$c#i} zoOjV~oUrx5#$G-6v39udybSN&(>VlhKG@UBa5WOV8aE#sb}g%%J{T_h)^vOrTh7$8 z;?P8=A;3X-Xd?HMuhitj3e?m|zLq$R7RR1v7>{VowY%`tgH_FBrrvbnpM>MrCkdmu zBM1P@=d52-YMFVk0aTYG3uiQ)(hQkr^Zu0dS=K%pM4~VC|6WW0jRte)^?OO6i+I;9rGM>$V z9!CQo0O^om`pXBW;!_iUQVMBz^{OU!z#2-J1tlv`LkUXwXYSZ_T^A4ri<&sjJxJ44 zzcAww@yqpfw@aT}ph0hWdd>O`jmPBNO?y=Ui^mJ+8DjW`>#enE7p~ zORUy7cz^W;35|$Uz{&Q+xL!m6rhYoLK>WpzbQlGay=d?1?d2~H%w)p3<1;B7<{fW- zqdIAjEouNy|BBfP7u=5bu=@d0j#oclRt#x6H)#9^{n-<=;0l+J1-2$D9Fxx@EPhx< z|FEO38m%>40#opA2-xlGHb4GuLj9Zb+&wt3KFc2OT6vLyUQ;-03YE6r2{_v zyv`-DH$9RAn2n1kmPaEDn2P6Xt?Ilv$2ogvs_RT`bZ`Hq_a^~6a@;kW{A!NmfKS~3 z+uS=p9L5jBu)|kb>D_Me)22tg;s@a0OsQM!B4Q1j@^1z}W3^sa4iFe}XTgw$!P66J zIyPY8IrMLN{j6p2E;x1rOY8K90mkF!D(7(u8y+zQ@RfZx-g4{(8YOqqf8-xwK9*oJ z3`a1rTc`r~fT`sInaq5NOm3qX+fb6=ehssR&+k^Tb1+{1%Yo@7f52u&g^)?0r zo{)3E(0>*W*Xsdf;{HNTVzwLoAjCK3fbSVF@e^G6WABPx7vROsS+m<5wrGu9e$I$5 z5txmmA&)}z0eT>TcUsm9K+}mKIs2~^9OZ?{62P{BvA+nr8Qp@sxYb6}QwalsEnB(` z^L!~EGy`jyNWTcvZC7K^&lB--pDCa-`hIl-lB|)!&CM5iG!573hyGDr)<86NEKvJa zmj|$cZsyq^g=rbsdE`ITGkiHe=i?4oLA>Ll{Aq|t>No5R=qLWAWgGn{p^P?{;VG5z zx(}nx0UC}JnH4a`t{j&3o>6E@K5dvS(T#0*QZ547v+iFJy#Ir`sQujfW7Vt9GaPt2bq62eH+S?6^Zw$w zURy*8?RrP?4`&Z~;X$5W4!SBk+v5ivSXC08Lh)BHU^FV4b=UKtr`8g)L&cYi634_M z28$!B5Wu+|=|Q^DcKu?e!RI$|?FPxAN4(Y|a`_R0U3pH!x2OlL#~)iiF6Ke-68hpD zA|=J>jUgL!i2xNZMEV;A(t3(?X>B(jX2NokN2o81ah4_~cdb}u(txbaFJ_DR=QKe|tF z80JKEw|Rnl-D{y^QY{iMJFS{M#?$pBXr*>!?l6qRe9u2%^wxF!B@H5aRxa@D8qT$m zdH^2wIAN>n$Ut+$tk-ij{vXQ|TndYmszPhaes$zdD`@N6_a9-oL?cNWN*B~~TCdAk zn@HKyuukh}2X|e1D?#LD6Si20a$j|Fe#qlZ>#9RAKE8{?7ssdt=2v$a7_x7l&#?`TSGE*J#cTG5RZkE zdm1(BBHt4Hfx+NcCP}D(_Q4sxS891X${{* z@&2Glfd+q{Z7hn`4bAT{lxfzkw|`iqPBrPBD9Q%xIqrm0c83Sr?z+- zyuR}F>!5S*Q(8X5$-wB-Um|DB_O7~ZCd^;XX#?BciyyH^_N@NNy(Omw!h zi1&_fDMtc$*P9V>CK%n)aK>jC-71oFkD1p!lkI=(Atq-%4w*$~wuqB4(CB5>o~Z+A zv}5#d%)lRo@87=7$QZt*wPE(Kj+)o9=hX8mo4=HiXnWsWo)M0ClD$BOY&&O?^a%A( z`x!wimAs4QF14#fwAY8^6sR#yy5l|P9yT-_?SIz7NTRgYjI#L_7keqMghiZvg8u*QqJugf`Dna@z z8*B)ih`cG0A6M^}LOWPcU-lnJPg95|cOm@36dHYp1wS=A;h&O*Fd)iL1xW%GPX`Gl zWU-OaNBqbOX8r*WlW_odFF&7K8d(q4QIOE3-8RAv8C5w0vC`H+hp6#RSi*%Zr_f-H zZ6cgCACKN{@F*;ypa|Idt(-C87FEQ^1|rZBRTL}SIw{;>8bVsZ1)bgNTm*jq2)0M# z54;_e(KsxnKaVKGJTm(>dU?p>T4Eva$h_Ou=4uqT=!>*}f8cpus4a8xOx(GJN~WpA z?PH-S!3eh~F4dDseKs2bv3cp3k3Ysy zpHz$c#V_W5AI*PAUX{P1PRB=FO;fT>p8%S(aJcyTxI4P-tT9EKh_WCx=`cl4L5*Q) z$L-uhH4^M~*fj{?@5KN6Xr5N`Ey+x%B;QO3GoePG7XAmM8xP#Yn5huE6A|ZZplG1( zZ!)EWzs|UhB7k_%NU)#$-3n`UI0<_SEJPjoW`eBXB%K_^!w&{f2kU_Sp>wq901GHl z79yI_fC@fo+F&pywFoe-QF=Xc$U8L^qAaj^bYPEVf+hK7f|w|^`m~Tn#%;F7&#;sz z*@P~TaKoc^nP7=`hf=Zah3681D;W|7k?VC-!2VQ#{gnx_LjG@ma0f337osXeYh=fF z*a@Vf^@zRcc`dp8jdm-m1n;9vu>F`2A}oILlyFq~2Gepc*Y3Hw0$&(ViDRS(ZaY4} z?QB3j_#-ZMJb+IU#^I3PVH5whxUD-dN-2^}9cb3?ooTXS#-A0FQGt)12^cwXY-z?D ztBHtVw8OOo)FzKJT37@s!i0{Use}(!tQ-V~K1r+`V;A$SSo~R05u1gs$5BA3Ic{yQ zOM7S2_GRnJx9YpA#m9m7h09cta;4!#Vn5W{XW3{SvTNTfq5?Rt5DoM%moO^d+1e%pg7jMeRisL3T`af=viKY%8p$m23*(a+@ zgj5k$`AT4wp~Z25TV|Ef+0<7aXhl#F#_51hGwNL1Ck{AnB2x1*M-qihfm14>W}E0Z6bt3{!!Dg$SuytOiz(Bj4lB8w37JrkskO z4DP;aEJwP5Yac>)2aCYgAN@w>${c;OB&>Fec2IVb|ID>r^1Hkgb*Zc~Emp6OYa$ z{-RoiF;P0kya#Us->VQy!-yrtQbGV0Q3L&sOo8LYaqz7ob^#FMgTS&IRGMcRo32)`2};Y!JkIY1Uc|UP$FVbJ%7$H0#;)j+|Y8A zMNDsfXl0b$TDwKrQ)*`6k5o(P7qbQ0be6@kXcPH@ht+?L)xFYm<{x}>s&+de1 zzBCJPAl(2?1hD5`01APIL5vMT8#de-YQ`%$s$qh?D@fHY^L zMDOzzA){nX0%AzQACKrdkIezz*J28($qj?C+NDy%68WQ5`DquWN<*=f&z@E?{zN0| zIn~Ljlc;2ZkR(1bW-mG2K$kkIewAx><5fjqNTLns2aETPtAM|=6&(e{w3ei%xBS5!a0sekvr2(0_rPn9;NGgg5yPz& zF{u=O(;Ih1s}d6rNVqh-GoNvry_4yfPA0=_ldt^H5|6%D10Qq#u_isrK{^mK9=&x- z%h7mi$QC8+`=>~9r8!dP?00t>-i5SS>qc`D^8M(u#!M`kCMSPSCqLh}N3LY1tTd!a zJbqoU4xkxd7=N~?Op-lRLw;xP?&aV|oFDkUfXtD&M?YrgRLD&VG5wG&`>9>dj`oS( zrnj1yewf@!w_E}9PwEF=Q}X4R zky7LUCQ}H65ft>V+6{$RM{DzqwV!PPIu?98lshoiyY9dvDg2H9eonP% zzh3%1a}jTxgnxw`zy!!nec+I$$E_{HXmeFE>^jnT;KVt8|{!y(Jo}!Pd>x zsf2X*hI446BkiT8-Kt627`B)Sxd3Awc3hYFBvCtYZs>zp93RjBh)JAoTXd>I0fPRt z_-U-lL!=_uE*COLJAt-~1MY}q2Y~d^yg0f!!kvBJ=acsP-}I+jrjL*N0S_NRbXlhW z%8^2KT{Gs(ediK+gc>`$4vktc)NOWZCd1G;2nDpQA8yZD^=&j?fXfsgvI8gi$LfT` z&GXiNkETs9HLte&tLZXY>lCS7$S}Tgp9Wp&KONOjr`fmD8_l=F_^q3H=Y7NdKG# zC70?dsJHH}D&H15q8h{p!2Ez^FB^B1s#!zy{?=xaG~w~P_TMb3r;{vu7Wmv`rMu`X zyPf?uh!5py1R@j7h(GhAtoK*V+kc3bv^Y^j= zmr5PT&HrA9mKXi<38}Ms5{^)zO-s-6m5)y-7JuK0P12j``r;lIFYrCl+G^>%p1`U` zD)TeKGG#{2_lU!LB^i+A-A-2kB6Q<^9WnIAW`_bgq7f?vzpPj~%B1eg$Fe-5H}s61 zn?_Vql1$WERuyDY*3rZBV+fT}*47^O;LG*Ex4z+>W{^Y578Ej*U`v=tyziuF-gHE< z_B54W%(QfwnF;$n6eE%6U#;2z@7hn@Zr)@9l-4UE($n4*sIyvw-j{ReRUU-wCCnV{ z3wpOW{uC4WYGcl2{;4h>cOv>X0V;TYUwJQo{KqRnj(US_zK%RppX?9g%ja;WXb>8U z_O3fJeS2O4OLQ`3*YElD^?cspB>Ea;#o=4E>lg3+_&*LGKAd|+E=c(mQI&jU`BUMF z+MkrlYBQkrut^oHiTLB7jO459U3`r zzI+a9d&mab!B&v|)_9usD)Sd+R6+LQqjY!X1MqHM8SP2eBff^I#)b2Wrog`Ml_wL} zvq75br>DLRhhDwV1~!>PBbQ@gr|2Ba;1eZqawW3uZ4so17Lr9od*o#j#Auv=_dfFr z^Dbh-q3@JLSk6wExJjC}Gzy9uGwUEmX2wZXJ+()4YKPog|YLPQB=x55>|jLdYY&{##ZE5XGY&0_#ebwvZN}ni83WB zsRUkWG;e7q1>^nw0(IXBOM6E>BWJ2mkwJ zk1K0zE+*E@hSgZ*25bf}L0v7*7f_k_Nj5vSbfKpFGpEwCJTl8jHLF7DNXHVrIx-7O zXldyd)~k^rw6V~ypaWNICjEVc#(u2jwvwd$=z`U}oU7~5(G7O>_k=i=2hABY8FL-v z`sTERqCA>u^}j{xYA-jf8&ehcC1f+;4xi4YbvTmU`^nYueAgEbf2qEW)}3-_r%x=! z>k>Xo8NC30AnUp(JVV6f50Mdk_+AB*`Nfj(B3HhRc=y-0=+mwDO8w{$hhMZ0rY+y^ z6mHJn>@r%$9M7>LszeP-p$!p6>eq|!x4xS+ycIJ$XL`5G&fcBJK{6LJcw0l_@`9@x znj9}t#CapAy=N_uy1V7yc>o&M0mC~I`O22-(<*88rRN7KZKVXx*cs;Gt@3F8(k7GC z@6!WO`?1LQlc=|HRFdNFL5#oRJXL9BgYXW&={y0Pzfg8i3IIeI%v=EvxVK#MRJBQi zoA)z1?z5x8C2dq{U@VZ?RY*N=!|4m9p}*)$&ql zqD5G?GUXVQMN6reFAn|s&eM!j$6((#uO6;uC+E_Oi|GRk>iC#g0|lHH^RoIGLTbr` zPPBzT)F~l&;a3Uwe+sQ96Ah1J`aBZd(GnP|?=|>_o4-W94U`24O}8W5x-|g`_{HeU z>%C$*6&SXk^+=(_J(WLiKHB>S%t5V*ceS`a*Ed$T&)2e5vSBW=*;a|{2F-NxEr@+U zFfh76U=_dsg=PW5YqH}Hd*UsQ4*d@9@azV~kOnYET?wI0hp;MoTSvVS=aE9{W2=0eb`T6RWJZo$}^S6c+Y6S)gr6k5#{&KO1=PsV& zr1u`_fi<-hbllS(x;h<8X&7!hR}caXG4U`D*TkU3rno9-2Qp;i=frCpUfoR+SGh%F z?-qduSc=~j(N}t;A~WPY2S}Qx^*w93zKC|*kX-f6h!3uI$)^~`mZ6Q!)rl;MBj>(* z=LMHBYF-@xAvlzpP9kdG4en*9v(R}bdz~LtK(X{tbz5qhRhXNe<-5jl%F%N{HZ;eG zpR9>Ak6j1!9NgopqyyG=Wn~NB_GTmJVq%)8_Op4M+dG8wxTW722mXd@^bVtL{9*fE z5Bd81eAnk&f9kP*PmeP!48kZ8k-n-nf~L;*+&)^w=5?Rx$;R=tatK4eRlh6H*LhQ0bMNb-X-ZC|3~ z$j#GrkK%f2)P8A;guzFhw(U6-o3W`;ebUatZqfpEKjOg7DCXohot zFJ$}sXRCz`yrTzXM<2WxCI>-Y%S%PV&+U1X%s7r4&zXza3vuzSiCGksL;4ZOFCC$;HZd8+(%x1opKl)LV)`{(v7g_OILcp!p(ZZP{ZRxR$so=WgI0eHW`Ti+?I?;e7%=i@1#bh^TNzn_uA^ zXYG8fudZU`;OptO2~Mb-Cu>uy=5IvjZCvr)gc2|~_}a{R*eJN}Z~cRA<8K$Mg(ldD z?C=duDrW3eu;=)(?!6Eh-}>y3{*hc=C;Y8Pq{U9v?>dSEX{O2Q&tZF0z?C1?Vtg+U=ngzkeQ|<&OiQQ?a<=0=+B^xM zj1&4NhNfDOnHb7dcXLv#ICar*Vq;SZr?ML!hXmPq%FmRPQPNW)<;90mI5%n`;I>{) zWbCC}H=qc5I=8n2fmpGQCmQvSv@Br!-QjlT`R-49No`$6`~D#Nwr$l;zD1$qI+a=x zI-(IDXJ~s5x(MOhwG4h-A1{~dSTMqU*Lw|V81-{~KHNOLK7Zon`*8CKbc95TNcf2w?ra!lulJTa zH2~Swo22dxvZczk>7@uk1??j5o1_ zO5KPErX7a2?jx@DF%mN(Buc=(5X_;5Ia+n{+`;OiIvT(^`4^9X(ji6YXyIw-I`+dc z6|Ec~oj<(hzn|WJpU(IXlxz>J5QpFg(cmS^ao?74jK?6zhRU2js>_!fMaRO;zd?t)^FYMMd8I)`*b|(n0Ck&5GvK~0!4k->wVA$ z$W+vV1nzM;@k&T~@wPuY0!6p)G$fd^fe&J1Dh$^z616M`O%P&BfQ7&`q-8Dch&Fs; zq72MCoA7Do>APcpEj17BlzaiGKv7<4H0ZQy2KGOruiZT9_CAQ~j{|>@k1%*MeC>SA zyfPGRvfGgIPFT3U7tQaIb0@~Ip|AKNDq(!6YX>Z+@R47^C+W%(pn)YGN8q(bia!-p zb(=vKIDnsXpXkA!i8BwwNVJ}s;f1)(DU&EYJk55%T%xRn%s?AO;RH0q*a@2&I*4B~fF z$z%O>H$!GjV%(qw0%~aJ@qfB(Y_PWAQLLKjuBTR$w`4YS+bLcX;^Zw9@fU$%{E;4M z&5|X=wkA;LxS~s~B?Yo#z$4|f2^V2UU*(WJ*h0uhb57hcH2O0g?N|U{EW_JUn489C z=i_tnyj`mYuXJm}f%dV(6h@!NLt9}D`~rx_zFWUh=EQ@Xyf1&1OR& zveqhe><2<$*hCl1epy!TV68hGE#fh-t*21aLmCD1sQwGE@xr~mN8C=kf0A0jsK zFC9E{vf=jspJ{%R&6L*aATo=q06WHJ$a{_h-{0_Hfl_N7CffxN`a@L=_37MPySt(8 zu0NXvl$NvXpN$4*-dad?G}?c6qbGI=o2b$+j#iGuzjUU2O)_=DEj(dAV z!^P)OD*m;KO3JEV@FIWn@o6|N-UC@`aFuqp{{JxdPT`S-jhc4Sandn5wvCE9wr$($ z*yz|+$F^6V;dzbQ zk>e4(JF!bj(HOVPT&_f&N|@jF8W~L*(Hz$z>Pb}yuE;IXgO236^loXXYPCqk==-Jsr6?D+o_o5JfG1z?hCotakBX|iT_k#QIaDy3Z z{N->6YVQ~bM?(%5vh~6y#bD=*@rw?D0vmRyF-v;SHGQrQ+{6|F7B{gzqW|hJKZ>gH z5)4JBkd8?h@5@mmqyuI@vA8Nn8?^P1T>05A9!oH~$8%LF91>%L{G)23eotOR02?Bw zGJ*%-Jn**Fsp2YswG@yzHm^0UVMJ9{zoYTjf!Pbh>v=NU*DFo|P6vj^`h@o3%_zM* zMYowmU57*Ssx#DSU?%Qs_-i7$uq)vCWr&#V6jMRcD$rOXNmWe zL}FkLtzlr=Z8hesF>NNxb_posWl!Jy>f%;dcWH(gYR^;@c!?@>FuA>B06UI?nlphL zVm*`)d#bMoQQ|-%2T-G&T7Z;&C#Vhu>6VXpmSA>GU&c-Q{YSngHdL4he^}a8whGYd zyLr3xC#rehRYvS-NtQM{iJ1I0v&Weh!Q8MKh_#}US3 z{iyCJy|b8N0GAvQFr?dR)hYTN&yz|1f_3|&TmS590Bu`ND+A=6$|%N9j_-KuNY_H3 zx;LCZT+*1wmsd|2B@c6MKuSEOj2& zY7ACdea<>?{<{^g-E?{k8az{&_Nzq`&weQ;u$Y07FAs&_VnY^C=3#G%TC6gRF*5tC z$trVW`Db0Y>CjSHbo^K{lydqZIW#PhnB&1PWZ#<1C8qd7%&G>q|QiqnAJ&zNqS}Y~N(%a~CzJdgx$aCs9BzU- z3)qa%tj`3l-CeC=z~+rty4UmmtRpMj86g? z9kkj~7vTsVk^Vk#y0(c1&rY**Z3=@}oJ@SKF;~N`RjLwl@N+qT-7P3rsd*IA$dM&D zOoGMznE~4n;L@=(WF%-zcB%t?Wm&+5e&TSI@&v@ZnflUQ(`S>7Lw3~LxUxTY&B78+ z?Lu(~E90tNeHBh6_?NEx>lk;>rFQfpaeg(<`tl&S}pSBuTCvdBp9i{ho3`yPJtS>{a%ZAS5>lbmZI(tuEU0w! zu95|NY`DR$VL+Jjgm1dK$Hp$?gO&6(EU4~0k)qTHLWIcUopK482 z-u<+U>qe@f_fBl&KR0qJW*y7kef03fN?B5?npU%da;S&b2F!!Jz*N^RJiFU~-fA~5 ze-V_xH?qEGqJEC)4y1tlKC!;5pjYiw2X|wCV(8AeE;pJ~^y0aIUfo^`Ig7Iowk%Mx z4ZCrZWlXYm;*V7A^V~&%RJY1qYZ{zcdnR_7PMPSy8+YP%UNd}GxiiVRaeB}Cmk|CD zuZktt?TyQAW6obms|{ueU&5M0-;o3AY=ea@&xe(R--3)~!T*ur_wxw`kW*z*06G6s z!GItSyauTAdluw8wmk3}nDTbxp!dn(C0iEUmB4$#BC41*3wW6ur+@cE@!e2GjH$$j zv>(?NdsIurpT=4z5oc_N;i#op&p)wR6<|@Vdr#Di><)OsyyP{{S0ZtxF@j_mdpph3 zHS7i4`##Ny!R5N;A-$HKCWT?lJxeoK`PW%O`Y`i9wW|Q8Pyy|*j4gD072!c0I#do0 zIn6o{_mX!@gF+J)@TEb=E+q(!SzacM-zOgz!qJAy#ly%fM0DnhOC&Xql7Vdbt@Co#b`V)a864ZE`D#FEx|Vv8E!)gfA_=R3ICNbTp)c%4&7$Rr zE8R#KgU>k>;q5@T_NfZGZ+W8;ZXbnV{nQaLzLzk5KK;Xgb8*%KgGA1VBWh=i7&v{L z1n!gU5*8ADrPATX>HsNe$}8fL!s!rtL0`wi2nfI=-{!(5zN`r4=$4XJ4|wg8%FLnl zNI-_I9aBS-x)Un%1NWl-Gb+!Ob+P`_MH!9%vsAd?UtiR)|8gcXzlZi?qzxHGPXom& z(kNG*=`DV=z^z>KR<7{v>(k2Q$G;(Bb{XfTDDHrWS9NC?a+D7uupnkhzwhUidZ^~b zL{LujkF6XFTJtQb_d&ZL3D7d;-m z^{U&DE}{55oYhWcIZ=!E^%(SiTi<(Uxg zO~jobxm5haSPsPUB|D#W$9GXyPnU(yevkF;&U!*F$)(fO8BnhDxB0=qdF-P^J34LN zeWxH=!N#peBeOZFAiCCnj69bG_b-hBX|4vr`Q*qwkw+1(+l%l(y>1W6VcnJn|$f;_u4?$N?dXi?HI=AswfzA z&~I^r&1gRb(T+!m1j(lRm&y6Xh;wWSkS`$#2hccmq5k_duO{}${-LwhH{!SG?+6$m zco%JLrWoFNA1*95@!6-Kr zDpGl?sAJfaDsOA14jASUv7KzNtWtJ@#FMedrWuu56&|YJYUoi6&mw~yGw8`qgTgZb z@E8#FLYo9VOvesJTVZ8d#kbBZsD6w;nbXqX1^EDQ6>@mj-;nRG5Vz(2Q?tKf6-|WP z*SE4_{^Rs-KZmMm^|#zHz-6X4qmeX4Xif7im_F@ zRmTc*WMxb@UZ%EgwwHCbj#rntZx{DVrMJ`LMS=Ij_|d;@HEcia*RG{!%pbs_z3&%+ znSNI|b=9ZWkctLJ@9ke;&ad4JeqE3NTW&D4qf$G~A9*DUqMw3mI5c_#7K(4ysHqT= zl3ZAlRFL{gp z6k1ygLNa_R+1j&Znjkh2;qTU&DY9h(+x7qDoT9^(B(qUs-?oZ>YX5#N^EbRZT3{HY zQH!ie#6*UjI@dUqdYek|-aN0vd_Ctcd5 z>){tVFTeN8o}%cUShj2rC|);?2F&vbUQy%w$;ZlPXjylQlN2oq?b9>C-jUO-!9$uk ztlK|pqU>tLB#!I`o)ESPY_iDy5WxEk|Bk=Ouxh>%iv~`>4qDP%*G9SE@B$0@+~5~Y z9mX{wuN`aQ7j^|Ms*_Dx<4@sT-v85%Y~7P_neDG=zb5!~yjwFMUMRILY788VQO|)q z#i(;KaTXT`41oi8tLXvWh|w{-S9LJpbCe+aE7v!tDfJQ4+hTxF$^jH7P4XtZGR)ln z7?^+;R1uKEGqn0b&%9V?SdH;`SrUHy0}bFM@$tq4h6 znmD;t0W?VZDp5Z<@1sUn2O%$oxrY#PJ&U3PF?=$do7R=zCKF&x17+jFkJpZ& z)?%Tm8!HEvLcGjJC>2&8lWWWEC>1%UB+@qm9D6*ZLOLP)KXq{>^_!Me$ktpw*%3Zc z&S>?6X-3Z@KRn&-_@thq$5bbs3tM$$J>%5U5WC$oE~<<**s;tx5-*v>0kRB?*6T6B zCkqgS-`*vtvd?}vw)$|!94QU=1zGPZ!0}IU3FIER4KnYU6*|~sz8Vlw_$rPK%T){V zeikIgWa|cby-T>*DcNJwO6jQ1CS@*luPf<{LXT8QXS#41hP76XG|YU2b{V?O(gf4+ zeypHCfpX=zdo>JjL2wF3cPWYc7S0ukra`!Ix+2i5Pp0f*$&hp|Mc!M|dYJ`yaABWX zloaP~wh8eh$ZfBHWE0!5o+}w=L4KlD&kOR4d}0%pjOc(tp|`S9D(}zaAOR>N1$K=0 z+pT3$!?~^gR8_O`&x6=|x3xBbC|Hk#0Tt=)D&@0 zt(K#Nl1co4w_kr(Tv}Fp!y-wSsvra9*~~2P(eQye7<_ibM%mWXy~9+{p^W4{!20hcs^e?<2=TXmFt$6ojj-p6U49-7C+ z;`TO_cleWCCVS7~B8rTq2CIk62ucu##kRM0uNQk!Yml+1WW|TuUoad}TTSd%K3w9z z{^K9ZkH2Mq+NNQ+Z1_@3_mqJR2LMWGBGpW@CIf?`*)#0e)ZiB2WoEAJE4KxafVFrc%2F+SO21D(5;e@?y%e$qT4>ji39?V9Dz3Gc*{olkalm?z;Were#ueG*Ws>Txn-z3YV5$^+W2L z2a@m*mN(skgb6l1RNiuRyOng%#X#ej8~URYNW- zi)A^afsslI6)-8h#wpO=G&ZhV_mV&7jk%Mm8-7nFAqoSUmtF@Ay_5blwxzr5c+-y* zd2QrRL5LroNbq64Ja(v;9$6**{)ktD{5N$%k7lW#qus|L+U+liCS=$=40IbaPnymq z-N&Ali{;BOjyOBqTwB>kTQ9WxZ3VW|G{z-c7l)gnZVmR2zTC7>Ozt`~H8hhg@RK;O z%67sG<|MDVd3)0qGq1oX=Jx9ux^-`Mw=UhZn~IpUdPFG9T@ZHa3#rbHk23wh!mVte zZHTEP(h(2ZAVO$CGSI~t2sEw!WhUYi+Oea_L8D1FW!~-3VY(OTVrCQ!xJW4WPCb6$ zLhzRPGJiE=n@R9E8NKlOkCP*OyPPnt;K8?Hes_hSO`!1UkcPWg01UC#m?eP2PJpok zxN-#qR8GhI{8?%>{p37q@@p^)K>-|o6kt$}EnXTrzdd<=_e-`{mWm*Z+OrGDR#4FP z`1LiGCbDvZuF(A9QjWqubYyA!;D5e0MR?9h)2!cy8f+@u5o&eD=Iy|KCbao234(m2 zAP7D9CW3@)Hp58#Vu7h}c{d(_Y`toQma8C4bi(9qM!4Y8Tn`${1J4ft`3tfi1}fh4MO6#csni#}t0giZa0OTv z_rbPKe0#s`8G6tI%+JW@#}+WqKHDYj2s;c6l1GwZ*3@ZAo7nrRF!nrNf-&`IMS%h! zXE3+JnEV>RsDJZkh=oC-UHu>u$Pn1`-!HZoXq{$2{4p@68vq8604p&L1*I~ZyDkug zTM&*Ph$}Drp=B)cSG}B(4wUDdd){iOOB4~C2W>{^qw0^xVztIh;Rel0cw{TvaYGqX z8_VBH&YON5+(oo*p1r9OWQDXYVPWMgXi0hHsUjlT+c~7-b+Zd;!&BIa$DzJ(8r$^g z+OdY>z?9B0YPn@}LKK8KNAd#{eK;SW+r>{NZf+@{1Q#PfH-ExWHXW&QVsx*p2$U_t67$VGoljUtU$_PO%k-(pW)!KlUHk`!uC3Gqo>@eAK!&Y;4f5COb1ml%tn z_{aPQh;YhX`F`+MrXui4yNNhKeOBL+s_Ov72nXq5j>I5NvKf2Ku~fofM7=jWv&Zh) z^bB3US|gi#_mQtX@MAzTJ^3gk$O`9&x@RI!t^jv-;j@IpFh+qXh+#qdEY?H=;EEWdxe zzCID~Sbsexf-d{qV~k`i6+$k@Xn&=@zHV{5d<X|*j0EP+ChQ7}- z4^?*Bj~_>C4Q;kG_J49&Uc_h)J(id(SF*c|FE?r(kna~z|1rp<=QcUF@+$buvGm|H z=Lt!Im}f#dF5h`caIvohw7B)7V2?Qfx7~MKEn8=Ic?-oly-L_0NewlVeX)V(_!G3G zn8sOhy;a8px*DPj=pghG{JJggYG^j}aL!LU1Yq8D^j0`Sz;fUdO1wQ|4-WDP?j<&RJr7sI%lVm}02cyzvF5dl%d5-V zFPCA^f-E+9mJ$FFT#R$*HN!ZUNV@kBWDZz`X0NHpD8a%k{MP&(9|| z4zQlBT#_DbiY(+zH+-j>;loa>FBo|8aklO3uXj)g*@<>s5ZG^Z&Q|Zg_S_ypaf_9X zLYRt1MA7ixL3({7c6Dtd3P6`;P-^ZK=%5`bWe#(f>SenJ8H zu0lsB?U=~xoO1Z}rtu(J6;nue*kS6iyVAEpf76)CZ)ul0(ja9mh;xbZ+}s$(4BEdy zwGC+oLYZ6AD(YJ{*N7Do1KMe zu^%BR!)k{&qs9GpW=n_DKlEhljt-al&$1UF>^Aa#jXS@NJ~o+=oB z{ROwBw=2NOx>B^8wO@6;x;xm^5nsOr-^J$_x*dzo)VQFBSY74Q*`iE8Zf4=`2mR+c zS0`<2naqnNYcXFev*Xlb81e#1qhSr*@rEJAq@Rt=kQ-md|I_6qmF1znH!^B-PrdsXQOy$HyQ| zjE%GT4A381A)_|Y?CWEf0VAr60GruXf_BA&psof-JMkCs`5z0VaA~a~C!-T%CvrBi z2cBSN>PyKESc6Tr^q$Gj4#dS!%uVZpuv0ypEFXfa_O zT+MrUY+0kGkLYN7U`X>`AsW0#_uPm!KHN@9ntGoON?qPI4xXz};I;FgLoa291iPOp zT&hgpPwk~ot)7tWY27_)vs;V=ssy*JZdSd1_U&XeuDJOTrt;UN(UIrOr-aCg3j-kxcqajdl?1S!_MbIuslt z(W>^?j4P>m^`_G99TeO;Q83O=BTu%Zo9BpJ5S>}4zHI;BVUmF?}K1dSN6^=kF-Wc-=+$_K2HHo{YisU-sIRl-)imBbKwe%=Ev4*UE>mW${riTpMDmHZQ^I+VLe( zs)j1GBH>0a`0Q+^2O<5umM&T~HkfqFn`>6TYc&j5<~SzW;J!dZ=n*U=IO$s5WV`z+ z*r!JY$QoOj?imjxx7`nz45*wkvFQq7%>=2sV;5v9CFjI4;2Csze!$ZM_r8~=FyG=7 zTbB*~r94UWaK6#M!2B^vB$;>W3Nc0H}HE*tgj)ZVMt+|^!*iI=9*F)-JzG;7vO zRyI}l*Zy}P+VWo@+VnFW?V+-G#r+4jEZ!yp*ES+ZD?Khj7>;cY$@he~D?@QKM#7%l zy&nrWWxUP6>>@Acc)qF`&-_20VAA8=s2O8ho$2oKO}sECl^qtT-0eV)oc5;7_yYvY zk*^YMm6eUUF*?#>g9hDJIi3p~YVvNru#s>sL9x2j4HDGvJ^1RuRra!3Q$`<&ChVQz z$k!efCu(%Xqr1xtv?&r%83r%attlg?;lTImFz;5t_?YQUEjoE}GWfVF9UxwkCMIkU zthhoVlho^$$K&9g9*w>Ii~a!Sf4ajF8;W_9q&Oc!x(laXQU>FwG$^${%y1o1a;}Ha zlka9NvHtnPoBzijuEm>i(^{p3LSDL7%Sf(>_p;AxuxTppTBAzJZ~Bv1p|jGJqH#-W zPM+LG-TmpIg0GN15MdwQKQ1CO;WqpC3bzMUnyWAUaxZ$c}auDgt8Qxe!1qh zBBM}d{YIiyhEJNxNOX?p(suj!b2aRst~};|8?+mKJ1>>L?|`;6Igg{%ATn;Lz%{+c zn*?AagfjgkjfjZsY+GxwH`99XWqrHeIfPeNV)3$9E94n9iGgX1`pGdTm3!6ZZM;!X zP4w+v7PIUg+hvOsO<~q*fQ_ninY?Jp@9E3G@(b0|{MiByVLArXXN-1v`|e<{x%$_= zmi620H{1Fa_Dy4uEt@)ljl`5o|4+IWDY!>Ol&JJ$7fKIgQUvlep6bmX!Bp;jp;{B4 zK6|~CI?$Z7R!=8tAlmn}>BIPEeMGR?Z)=g8a4Ka^qcB5$q;?WxVfl0mbdxu_aBu*tEh}%-6aJUN)Ck9DYk?}1K%M5 zuq%5jiM%`giBoO&_PtD@djc$3uIUR|0<&uWd4YtP1PMySX%YD1rJ{GnA;!l2^4KI_ zC&QPZeZJQHPFc-h>>GTN{chze{+8k@IqPquI`iT1{v$Y5nEnH>)>;c$)C%M&l4aV; z6X38k{L*x`Ueij$`_%}uRNjCz#ibS(RH}Klqb{J8oIM+iGK|xfY6CUslzW_*khV3Z z$TH28K7Y;vT7Kgr`m_IHxTJK94Vf4%fZc8Whma1m5KP$#m!497Q!9*1wzKx3)K?+ zN@-j|!IRu8RzlGe9*>HsPjdM1S@;sViebrX==}n%LH^fE_%-;D1Ydw5z=|x`4YQRp z@DM5=t`@Uj7}uT8n~{!*JS~Be+i3arLXk5JZ&HF1PESGh8-OZTB8;2rw`~Y{NQx<@ zyLJuyFY{IJNoDgQ!KwO^WHIh~6qOhb3s^Ox-KW|o5BMryMv~@UPQMmW1%t4a)S?5T zL~i3gpV{Jm3*qlHX=5|WqRDI< zOk7(w2}S}e_~mipQvdyz0tnZ|A7;KYiosik? zAG$`UIq+ZMr?M%Y5(3*-eH+MVE!Yj-~{#%(^-sMPe(_hqE6+Ujs#)8!Si}3&m8NvyG&=ut8G;hxCg2O^jlxJ1Zljzxggd!tQ98 zV)**uc7Jd5=H$hUSFP?Sn;u9=_u=9BnkbVkfw5U5@=5cX0Uuz74z~R@q7O zpFtb=DUT#p6+@W<0XVr1EdUGnaBQ})p9PJh+*cd#u;MO~&h|D}!fhrG8!}Q3e{O~K zg0L$}s|j7>g=S)F{rV}OY`iZb0x9g5k|vL}uvdF~J|tI!b*F>*ED@Jdj;WLAY9H69LD| z_CQX!os7f7aEnXs+(!Q<#FM(@b0BYuHf;se=>@t%A|m}Y=w7@yOXIGj2VEvi6ZBvtyF+YGc0vtV-+jl%oBS; zkEbrcC1`|co*qq9!50!MBwe%g3GIEqc*!W0-3xgve;FA<9E2!XyDG-o(eB)h)6*~O>Ew17ZrlPQZq(tPJNm;BeS|X1z?ZB| zQUWCZNJkq#TYKm$3z4?6m#m(w>#&YLG$qOq+5qh_-9j8Cyo6`Nw8?kxi41;ZCFU^9$hKwh;#5JQEL?f zQD0Fkb8mQ`{P$EzRRT`7&n=Gog)3mXY|zsP7!iQ^qwlSPHA6}PH$vqjo-mjr{&Q$R zv3q1;k6N=oWknghZvZRrn=!4y?s7w*a9P1;ytWp~JGT9K$*q_v+zBmRQ(Y}60^PX+ zKpD&qpDnsFJ~Z*NH9#1bp&P?+dWh)o8*o*wP)8M7@kcYqcRU=L*Ga6=vdI8>;|)qH zGBhaArs=>5P1)=((`$L;%dkxWdhFVm;&|f15cq*apysLxFd&|Za>G*$pKSO z)s*-8O?XvIGrlYvHf#ifwGCdvZ=FLAOXa#o`1zMB+*b_eZoaDBZ!!BWUnF z3JRpJ#qliLzo8=F+l5@C1%asR#i&@8@puCx+=$z7_e5#9L9GRTia}CV!B5~w>Vahr ztPYs;4Z}3c^{-5o*_t!N7^Ge@3w&n%5-&MIA(Qwts4O2bg_F3dnTTLwsm?plBD9yowfV3_nfk0XB3fAULQsPnMQX%-fRVZg7N&i*pw()XK6t^ zASSWM9N!|-t>Cqi@4$I1-LtNG%7d(uidsfN#%_+Ls3a?49H>M3ZQFiwGq~RED}br4 zC+Oa;3JF4(Gm9iprAd#@;)@r(YqU6ofQ0-CqT8xxZO&^&pIL|fvamHFvMuD#)!w~| zLp3DlPm)N~1@l6LMzFJA{w>$)hH4)PV?I=s5p{V8>quuuGwdT;RX8i+r@BT}VBThu zd5kd`W-e>I$<}EoYaG5RYrV(kSQEtIi(zd*Moi01al=Bl`i<~gAtB9A@cw#KF_vKc z5~2uzN zzvDqj3L#pzpU;mvPfD(y&K$Y0-suo{XtV<#HM!~iSUvscEsr!}K5U{TtI<|u&p8S9 z{8y}c_uO6zL2ut=un&{~76lUdrN@S_7WcTM^5-+PoXvELi`G}%BimM}pCF>m?SBy6 z6&T+&#n$NIkiRD&Klb?N{gE>~5YjOGQ6t3&Nax(EjuZ_aHi0{(9b^ObZ2FbwX12=+ zXzh3`o9JC!-!2a}wCr_(A9pIyDB{#A(I{FPVb=MA;JA#)J-7Zom#S^J+pbXIn1ixP zAT9VW9{O@pB2n_^A<0m@_7kuGhF%R&OZq-KMom}Qew35BF_xR50Cl26cF-`V12HAx zD5(i0F(;8Gn^tTvjjcjzpWaISDw5PILqOwya_7Jkq>o-lEiZCEBRHXWqx()6L7;%l zlNX;VjtK@#eqfWz>H4`yp7N_YLTQ8_B4j0Ug_5{fHYTE*f{zhV7~m6D(xvXxn=@~( zjQGtrz-R#1Fr(Q6b!k_7L6ocpvLb76wi%n8@<_fL{ijaD)#T}+EV(l&L>aUJ$x zRe;u@p~Neq1bZe-HBW#{HNc>OV&2T0)|Faqkk+LhKc4zJLODkllEo_xO95LaPW3J~ ziE5su-xMWs2d+filgP#<(rcLCl5kz1(M2{MWK zp3gzhHwsoq4Ze;~kOTza$+}^y$lJjGH?%xAg0#;KpqL@%zO#c2{9-(dTXF@~hx_%+ z5JsWKFUR{*EuU}WI5#<2UOG3afUIF9ATf_BteDla$Ec}`lAI9+PqMaHhCnsDJCzu+ z6H_N7Utfhgs)8ZP-V0n}{+OVd@1k>sEAHr_I>aoaZCj%)#B)PPh~mA-oLP*tIOeX7VPPdbD-& zXxjvFg1g|BR>zOrlUEEwEp=1LUf4`RwLD2`=A)PRmX~oDZx(^edKN0w{l1VViSEPi z4tLCvLIu-Zo803-E=yD-lcqHzvmVd*p}#5DciWm}_=RF>=v>HbG|iLxY$($hq1eBpC$cD@GyhS>PU{_CyUnSLcn(sN)H*CtP( z^{Q{pS@g#PVN&Ay-_u&jGHSInRga5gw$&QZ1eH5hX`Dq$?I~GDs5Mu+P|onoVOdvp ztr6aZCE95`Lt{t2`r7;zNxu4rED6<3*0Au7fC|elPfE)2OD{Xsd$PaVI9wLJ%Bw5TO6Yzwqtm1EC z(YRmYQ%PnIyKLnF1cj)D+gTVSS}Fp(dc8b0+|X9VDT7^OAy1>wpfRao)|c6Am$MOO zsEsgx$a}W`z|{2V{;^3Kucu_O9HqBWNM8R&Z!bnVXi!iFH=Hf6STu3WW)g+*(`gGpP7p)#$9mDcT%BdToS#yZ?k zgsb$l0$t=|$keF`t~7Da(G>A=dk4+6=~cb79LPsrin=d9jDwMNqlDzB*2qMZc>gO& zXMuk7NZO*skQeMTRrDfkxFUlPlHO;^0^c9TfXnKzksNQVRM#rjY{|27lpFZ7v8tnB z@N?7G+g)%#sJ{r)WS~rWc=$_v)U{+*5Rqv{E_@0w!pEF91^pnv%Q$kULjD(Y zBEkQEAm{|li+Q1|Xaivr(a0j+!i-+`!=Sg98eq*cAw#&zlZF^Jj&Cze`yUnIn(kK0 zmyrIPE13`u6zL10(PRagmm9N2d$F+co;h%u`! zF8&7`;vnijiKoKwS&bdE7_c8qqs@$$@JmE>0Ci1+ONQFe+~s^0wvvVuXoUmo>086uPmh?WvKjzmW7 zm)UZ>I&fsjw=uRTqm?=sRtzdA`mS-2Lj`{J3Q5UlS_P4%w+2#8+Jzp(Zoy_e@{FIL zT6zxIYVU8C>*;)@QkGw+IUV@AC;8{-q{y3FHl(T#fd;QATN-r~JcPU3%GHd1?K|$Y z1oU0%|KU&cC{3XddY-%C?Y9+OAf38lL`g6x)ewTqOXC}-{lMWXrFezx2#NBPW(#~2 zVOjJQXKFJGUcR-ncbn$-{x*ONN>x}QBd<*ChFF67#bQRf|B`S>-eAy=X>+Z}X(X?V z41$9Sr)_pF&&jZ&UUN7Iu9|A%1ZEAeQR_AN|M)0rEmo}`>kPkE|lnlfCf35)F4 zgdI&1jl^_d$W$N_BIYk;ZQc;K5xyp`Mvi__^n7-A4+X$J7xIf^(2DVbz*x3gH=139 zp{jToa_>@}yoPA?1`CO1VcSH!^mxp3Rx62XT=7uN@bMVbvZ9ODi{*FNb~-5?;lvV+ z-cjbsgp~1k&?3BE(bnpq&1eD*aQR5*E+6-r6y5H>nam&?Ax-+>Xhmr@jdwODLWILa zNm?~`+eE3a$f6HI^2|rt|6lH4=)XOz#DyX_c*?)$ERwrJEsgB9SR3}^%Q($0%f4cj z9}45k_>Gdzhf|VR!_)Nb<(G8zL*%5}d@Lkn_+lv%448% zEW&pWO`&7BQf(|9;C0ke)VAK{E-uFrh-)=<1SP9*N&XHld{~Sv(%W(TV%_pJi;si(5yqv#!@AusxFIjat zUz3ib+1r*d?Q5*Mbn8qxL(aQn9K14Q!xWim{gc>+~t=bp`VI z_wYm_7apjFiX-IU|=BkYfwRFM4NVWoQg5k3PF! zBulyncUB+KBuBsjy#>~B(N>I&qpllShI|S-FzS|%@rce$F<;|~*Vjd|0uxKb>Br;$ z??u^@`Q3dXREw>2m5nQpK%zP2Z@wS|-97=KKdbJzviUQ!mhQUT5y+sfRdT!9O1>IT zDdD1s!h_Wf`eH8QtBsvZ1{hsBNEKwqw!%lO2wRJ}jr@d4?USY-AcELAGRlfn#|Oq= zBT)zIsfoL^G)xYWr3{KR8Mp0e?Re0(%J#A~Q}9)B0ft_nkQap3#@;UZ**xBK^M6WEBKXXPV5yWAT&$VXyE->0M4F znNt~8NIHMUXjuOsajxq2;54}uanuC_eBu=a-QZqNIv)RM)(`l?WC)$HBwq) z?9Q4T^ipG8=}8uB5~mKnEdt~rV`b3=0`y~hTt?vgJNRo3JYCFMe>E|@u*{i1%HEtC zm^U#OwTsrb_R*xIi`M7-wVtgi7OU?LMi?~I+Eg6*gS~L*gS{ZRVk*?=6|NK0;^U4) z)il8x+=$_zLrkJ%TRQz2gQJhaGKJPZ&y8Tx(}3NCp+H^E->>u%Vd?hTw_VB+E6V`j zSlDjh)|Kioul!cL?wck*a%iyZq-La~vO^OtBM_hjFMGFrA8#{= zoj!Deo!&QFOOX$!=7ELPaV=bF6aln&55M#=eyd2>V zkUS|CC``0rlmFwES(a&D9!XnH&to(&>MiDI7qWZk*Z`w7w>hel7+pp$Nf0U1NL(FU zqS#+t_5K4^q5KgaUWrvY=#Ty>u4R`I-!0`c7=QDK{htO%qFZa$zX(V8w40OA3CVo~QT{sKsrZaN-6arOJ9-qg zvZSvnnT<0s)zr&lml%U(;y*mN_MY0w{hrs`1i5x_qgk|C_stsh)KSH>>%t=OgOt+K zvz0gZk#$TJy-J};bj2#5ZqG)Y#3ri2;6;25mN+4IO5@iW*uQc&Tpq^KAg&ko9cOr`=+XFUeA$2vke7tq1FvyZ|1?0;Q0y%+U+|~IJ%I;{rrz` z;y{+(m+ zpYBFm>v^Md#zYg^Jp8A;gj>BON3U@WHM|1npy9v7C-63gVunL@jVL{QJbXE@9-89; zb|FEphR32PgQB4#E+S*VH4H12E=l0Xvn#VIq zVZ+Um$R!Sl)co?|pHl+zU>i7xQaaSo8K!c~KOS(FQ;CO@&@1N{xo&x|{;+Xj8wd7< zBYCMH16q}BOzr)K)=YO>Sin%mJPAku$lAEad2__ruvlm!{RSbJ%GeTk*=e#``0cIX zD}qL04c$YnG12=7o;77t`WU@x)s^Ri*S>iREM$EHi&y&N8I+0$3kuC(o*curXCTRU zHiQ=CjptfE#@E8kM*EK+;(0&#M+OA4o-W!ii!WUs+O*L(O=Vgz3hkfJ>JBQ&wE!#3 zP%cTT@8h5ceQQMo+%W-_6iEf{YH|!+)B{FQ4RwOyQ%kyo6Z$;f;bAv|Eom0%JL74% z!w9V}EN7RsZOzzzd@LYlG-@Q*4+7iefwD;wQvBP_v|w1ZCrT?4gTeQKIu+X6nejCIC8AsCUHYV{cOrup`K?iUBVR9#Ib%*+cwOkF=db#wqca;Etxd{z z-zvk-NY98{auNS{S`EB*J{kM>&hQIy2ZwjIryV75Nzx`p2+D9pkK*jE5O2C8$f?3} z8csPC16nlxUnml_sT3Gt)brdG4_Zn)c+G+xezxn24=2;GEN)n-0-O9;Y^Z?cZCQ-V zy9a-G`YA$YxR&4$0Q>sy`7o>ZeU$1DBjP(@9c;(`;oO-Lz6)oSl?(5OZH9Uh z=N44ss=a7FQkCOLTQ?@pc-0=QC72N>PR_6)S9V^BgRO`=8awXJ&|(D6&O?gA(e*h?-hn`Tp_M1RK3wdGXRQpCCaO&BY9IDtQfP> zgrd5FKj7G(%MRpe&~ecvJ(3GP+_hWQT8_zS;3(C;5+^5CVNBHx4T@NdwqM~zyV+DNlM@@871=k@GUhlp40Z2P91zTfo%L^Uowv|!rDTsX_g zZOG;AaOMz2@z)Rr6b#f;P>bUaT#5 z$NOQpLQq4;0;s{2olxMz=w@VYYH7J2<51lwN!cAqkm2HQjDCU$J=0aSV;l=v=j+uH z=)SMo#|vk<4MN@~T%*4r5whAbiV71fIs~Cwdtwk5v5Z7|PBMcjJg2S+siK+5Ie<_Y zyIiUh-Ikbvl!jvTb&Y4Lptj(;aBTgFme_Alv@x%1Mf(wPa=bXwQ>==-pzF1}a2_Wz z;lH%2?!>HFbm$>030Gi*al;fxdxjsE5}4o7E_IKP|WX)kF6qN16>v^TdmufjvR zibG^IGvZNZH-edKrn9^|Q?=3;-7Q9Xnmy`1nWV2s;fx#l7mu9H3Fq{do>q?!op^!x9gd|zP=kVXe;emznkt5X*o3dT1$#5_nmITcW$V2 z{x!1<-LTNmno7o6hHS(bS>l%E9*5$}^ z)aKPTul`r>z#i#QsNht`CxOGrF#f{?AME|(Mf4>$$&`Vs9~}Dio98mqw_kXKLf;9}#WCh=FCxYFznFW+7+u1xYqxFNwr$(Cjn%eo+qP|cwQbur zSNGR@zh82alaurN{ay8>a;K7~<{a}HW5OP%I0HRnc2Y$pk0G2AsZIhNk6Nt#0mhf+ z5e?~pI-jYp>o70;Huhx^i_K_!q2E5S1`2N9`F))*)3N%zv8iBxOfBD>*5!~wb#8u| z6H9$SRX<2N9da!W&e{8z(TjOdoK^wR*g6FJmd+Ecdv_CVT5`6NkQWI@{ej}m3`FNd zXQe&*hR}bu#~Rn*);Ku}K9hRz?V@TQ51&u--m{)C$tiHqfVlo(cL#B;j)P|{l)K<(n4ME3i|6*6ei=<4AG8(~q;?U;Zi$Szf*hP( zE!l-jR&m8apWVL^jq#&`!pa87QdhFULa>;%PZU+O=LDsNO^01bEC0^S0uPHOVX7?n zg*ow^0^@xRP8C6^*fODW6{`uvMy%xYEpSN?u?g4@bXyb+&RIwP4IBl5P#<`s3-Vk) z!C0Yt(DNS35y4E`BkWJv60a{6%6+J&4*aOfF1JT&xDM5VMvbEoZVQqc2LPDlMUckXfM`=;O%d1@S$-xl_kB9s3|W*j_{tMCXIx%@2NQBQsF zJ!6WXGD*~Ka=cJV4I#uNuF)G1p+*F1G>f{GlnuA#uxF@8&WrMeID*JU3!U9ut?`FU zQma5yQi{D#rKVSbSVhk3WI`(v?Avkqit`;99pBED-r(@KzBm8Nvj{-L#kjRaIh*aS7^?7wjAK&NGr^GJr<|k^tEsbJBoJG7E8lF%# zLAhnta#}j&>d@Rc`Zj}%sVbef1X>B7$lO2c4Un5;rOUixw%k?UiftcQJul2mc}wxK zoLRpzX@>$YUZ#x{OvTh^`ZfId=oP z`+K)R&QlRF8rEMOQdYlT{$%D~Fl}1Jx#*RX25Va4zIfu@#JX$onki(S_TQ5e+q431 zGii1=ui&J(vxW3|3`3MX`^)FIl_=XO?p1JH50j&;833gd@S6Td-JTRev7D~@c!NgX z!Ty0GdXIsNS2;wcW)qAy=pzf=1nPUCRhSvBvA)yxk+uAc`~8gxvg1_?`zpsqg(l}OI@ zPbnAD=R4)(c?aex;a__FdovU?kmoh<&ZvxzRe%+s^^;?qPYmI#WoHmia5jtU>%5#D zO*$g687nGnXl2e;kU8kE8&@bhVVAS|eGE^8h=2TBi5f&vHFA?2NGDA@HaLHH1$oP! z{lbhtD_Q#@<@>+icMWh>0+1Z94M?Q-p5C>$P)7 ze}__TJP6xoLIXLY7+@&JR(zKaal0>hp;L zos*N#jQ#Dc=v&#i*0q>7$sxP^dhQb<`=q-9a)p7QFSz0>Vn+<(y=YF6%tNa27cWlI z6L}(2$D+NOxAPB4%&H@}q7e-z-Vg1IxEN;5rz8475e9+J0)( za54Ck0S;4qy75qL3ood4@v!#luRCnPxkxun!kKa|@)(#fl(OWHM>XX%Cla5A`UChN z*UUOORcAm%?1glFK3}f2A5ZhL(9!P}<;h!pkR-}!;1L6TS_|z;+$<)}vFKFm612U7 z0Xm(@&l4D=4oY*PRN&#T?cMu->~4EdLnKgD6^xa_d)=Snr>q#y8FvGZ|Vumq%k zFY|8X;5;z8Ah=BYt#5hlB*h#$u)FU29BD@K>GkP=pr&1a$RAXZ5(Eo4D0si2S*wX_8*8E4@|UYwii3Q?h8|58!0 z;0h}gLPgCi72;puEi9j_*q5^jjEa(K&5ndpRXOWaD3}?$k%prb4G}xPtZwQGosmpE zE6rX^P@a#Q5(_VBvV)8n^QFRKOcLh&L{MtY6zVSEHp!tB8Mu&P0fAv4EMKKY673P&&&hq(`4?zAm9>W+Ca50HI=*G8I9VHr`b_(Ug&O8a`a2 zlwwX2R?HNdP*cu4wG26tC15S0m@r_(RS2lhUS@Q<5_+FyN-VBS+hrxNFL^r$q5QsY zoo`FXWo-MHM7FdN{g$(+E{rXLiR4mSYUq~5mc*5(+?H3K?Zt*VKTlJ-LL39|S{?PkC=CB=o|v4#)2W==i?rud^@Kg;$9A5>%hTRC78 zIsB(B@@-bFB>X2ZJMv9^xR2<2gv}2AKCdc*TD&Ry>k?@_s8H>@4#tat(tHqaY7~bA zavy`jR`89&mhkrx?IHB@O8Vw61~%p2>*F8BkeYq<<$mIq63eGsw@#IO=!HgVjk#T9 z$w_-w;wz7TsS8M(gwBcOhvkfz1ig z2e(a8QWgE&+#`QZT7)Jc2jWXg!3#UfLDYE=f8({w^@SzyRg2RWpMDZ)js1IzNbn{@ zFu)ejypM7)9G+22VqxEh(uN^@%u!IM`lAB&Iw^{~7@Jg#DuyeZ`fNpI)@V;zW|Vl0 zhUlC8J^@S-HK>ENCZpuXrm^mA*e^xoGmZ6Jz1Fm$1un8#o^Iagpi!q`u@0b1g|QI@ z8M^Yi?GWGjR|(2VX*Ev~r&$K1iG&puIEMFNV5Z)8R;a+BZyi6{C&|}vCbnqKTsBp6 zgfENrMRVN5PR69tBm;yO|7T|0d(&Deu;Bdgz`*VjDPIbUCX(-Kk9g>aM68QThbSqK zjv#Sqjs@mw%U*x0ac7d}PikmVqHfsneeoII2mhK|9_yA$M%s?leCdO5OSHXm2x0np zWp`v0t@X9ae&x%fWdO`kCYRQ7rkICR9^(^QGKibzkOJ<8{WVBZm9rvB0>Znb=As5p z$v~#7UcmrCX{4JwPppB->iN9E&aw-T>>MOCHsJL(LjDBp5kier-`OJizeIGOFE5mf z2xARNVdVyaBms;LDK}SnIhioI^DxyaONt zq-7ux>hSN#lI9S{4JOHJJA8g!@9!F#EwH$$8f5n$x+^+mnMI!$0_$uZv6ZOs&IEqP z63$hbn?D8qG_|dT4GH*HN#H{cIk`dL;py=0<_5do2KuV)@im~n>gaSwJnOqanLOg& z2#R&eBj6GIq>dft@PJIJ*3o}AT#INDBOnTDRpa3S0P< zelqjD9?li@vQ)Tq9vVV+?y2un!-g!uT8D`xbTKXxTp=of!oCcBt$vp#l@P9zm_e z_m@MVty^fAg#jtiIH?ZBJSxk_k1B~*%|Csr!0$ASeM?~N^d!rJ!DO7##h-;8)|a{!Fsau_8kK= znk&$AY~x6Ghr-|L8j!ANcIG03xUr76A2s@_Q$EudtV619H@P>i9AD(I1U0BsFlj;c z>MA#TOOx)vJ(6BX{Gr+sVkGTABxA+&5}_qL^5FzEym3yZnI{)M^BxoBkbF*BYuxLca48T#<^TtpN#cQG(O8f#2?3#RY$8Cda#~PF1t5BU*Uj)ovKu#uTQQe} z!COEs4A9Bl2m=vrsW~@_VDJNo9j$<6)XtPMPrQ|Z7P?1Cum*;EE({YOos`#apMwho zx8i_6u-lw#5RS)yfbPuxU}Tv-@b6=gHpfFXA{J^NEu@8_nw0eoJE#SQ@xjyQlFy)f zKumxpb5Mf`1Nt#(g?Sr2wz-(a^{mjQF(u2Nay=w_^?@;FwA7NPbSJ-0N{`?PHqj?U z#&c8@(6Z5Q(XL^)Vzm^&vvGT(kb#9ZY6#_6?vP==9TL^lGqq-v1+Qrth+4Jc;rjgC=<#?N*0X0g zR|D8M)@6ImLxc7G%>NU-Nf=7s9hs$HyGNwL8|bw968|WIy(V;PrM%|^3RmlqdE3-Q zHM^H)WGs$kEraE+78!$DMWhXb%_pQ<$w~BHLJGc~Su~+O}>;Q+1_>yEfRXpv;OCj0|5K~XkB!8G!qdrRNMW}ga z5kOazy!l_>n!_u^Z5DHeiwArPeacS&rQ$yXA>DLzvl@} z1_KE4rFRc4e86@X?P``6j7g`{yD#nfZ3JGx| z`5!p!&!N9rhQgOH*qQSjOjsKM^J8s9qH7V}%O)SR0}&5~pXTrmsWFWuLfJj(2ifl?iGTgQ|YO@b|y;!d@MqC;fu{#4V)ZE7g7=CDw zuOO$(y@c4p5={zkoUYz|us%E&y!-!Ktt)`+%h3b$GvETw_p(N5=Gn!4hYUhJ!lg+CC!ox$C1P%%-DYVY3OwwPVm!N!^@a3q?I-(`hDX_4+LoL zcbGNxHen&sTTeegI6*>B3?R^a`_%)7u+Rd$PK3u5tI ztWfGx#z~B$s&_|oO)so7Ou*Qmv+H8G!muqB4Y4#V>a(az&^2tC6gz^Kx*JthS?b(~ z#ZON=8JaXvw=G??%WB*M>?K78Kd5!vQn~Dcx#=Gmf1JAnDOkqC> zJ|2pf-`=*6+P!JG*n^`#8{>?K;Si z$C*|}K9Z}a56Jx*)}q5Ytaqj*LtU|c<|xWEx|+f&+~VaTE_E{2fvw>OgSg5ZbcPQ8 zFt2;%u_5*lnxc{_2jP0j9UHVXAnElmC|*y`t=4o-dgLoe*M4^>t`;5`FEfXZH_rh0 zFB#@vl8p0y0PZz$GCeI(~qDP0~0I`ixAV7L8f!c8R{>aa9oHniM}%8 zTL2~)vaf6ys(wnPLW;Cyn5=Kk}ds#3f$wa#~%~si!U?neKuO zDwqx$lDyoBr|V;V>{|`W1dCk~j}St59Uyg#2?twx9rw(Vp;g+l7RS^T%+pd@EhZY= za*3&Qn7Ld!J%NsTB+1G+HF~~|V)ZeE@+p9<(T=A&xlrc^0ftGs@ zHw_gKzZS~8hvH-snvG6*&C?Xk(;lMmTO2Vy)?sfk`$>3)io!Ll%$uHzXOo4xvnDCz zUiaiOJ9MmWg~>q>a&JPl5Z-}OhD_%dU|yID7?BOxtH;Cxqs{JSM75`HxO@U_4V?RE zwDbs2dHg%5LbN+VxR@3tS@f8{kurbNOVmwzMVXZjC>Rf#ILzV=FHue~TVO_@c~V;1d}9kB z7OZ`)RP($s3%izf_dn~!2&9o>mK>_uuWel8 z=YJ{o#a9X0Pw=<&8voe|;o(p2n$Ak=n+iWOKUCUpB2tn=Wf6>wEb+CF>{JP9*^v~U zju_*)kM*^+4~wr^xs{rl+D-Y?0A{*w0bIY)@|v|AZV~#h8_7f;0hOi{M;(^9oDKYQ z6{!P7d7jqUZ)}GJcL-P7 z*i}SRXdkUkIPCS-U0q|{x_q~6V=@*uYO-Ww=y5>iL$4Ftun_$CR)F9;uW7ylw2n|-pu!(wXF zki11yvrxL;kgRGzTwp2&Sft#hAaJprDZ%J$)df2j+BKJXB5V7I-7^u*-$`V)R1KnYiaU&k&B>Vvy@`vsnQSlCW1E?)GM-H;Lewg>uAz*V{ov&w+SE za=+q=EFk3LTvc&hlwv?9AuQFYdU<>>uL3J2%<;m#xR^lxUVh)rUMsg1m}>#&n3x1f zjxBi|3AhuK>>!U$hI{bAGg|X0T`w(ji6Jz4|7zQ{&dYBG@ z_0gR<++#~pP@Vj`-2856xnV6vAB0Wg3gd;VIa3Mh@&R~~DM0W^Y z#auxfGIZl)zvPW*rL6t!Dv2|-zv-b(0y6qKt5R!zp!7Aw-A_KB^^3Dvs6Ia@`MWD| zt1$8(XJ@t3f#mOJH#19uEG$rmkeFUkph+`4pk$_bj11?~O+k`jP%??Awf!WZWHun_ zOMXNg;qvEc0EEAg@AtP>kOckTYKjfcBtUq^GA0U7!*&VpaH*ymOq#;#2oxG5b3eiQ zG3!cpBJ8U81Q!u8Dyp5mnE;1+x=40&O%9U&s8)0(iEtRmbDlo>4R$EV1k~DigplBB zW~4#@I8ggyCrW=}Xl;c^Q>ONWnG*jPbTDfWjSdOPV}*RNMDYNuOXxq|cnJYj0x?qg zp$JI)(lI-}9R(pie(v7(_RcCed5-%9kI2ylj(cCn?&Oq_UGk%Upl)Qy5EiJHNYF zQSU?;E2;LzuUaTwRU4`dFLMKQQ6(x+x~L2#Y`kN@7v@$`HP^k1lR3z$C{AE%9rm<} z;u#Z_SH?y|vs>vPRHj#C>9N#(Oya7y%-)nI^qL?V*IW4~h$miuwGDD?Q^8T;2d&1X z!W3eUs`Rk%_u!u|ZA0|jIb&2qW5_fp20}K7NrVxtGAEx=>z$NebdQYkIc)5?wa zO$yAot4zCOH^%co+K$6WuT#$zHxONWA`Wy-AvaI{DWK{@+u254p9an zF3P7A$4|e2TP-#G^VdCMqfW&g{LP$a0Yf$rxdblE4F575Y97&jO6Ymq>HHIP_9cYLMx{Y8vT~Icz6E8@CGqT&7}y^0$eTie(*GJb)MqXI;&n zqzTz9NHRG|0W_Y1yg(GVXbv$XR^~W|Sw4xPnTje?*IHPrXS@}ehL2Pwer_HCE(&H5tU(hr( zRKl*y*6u--Y}D6O!e-+(f^U4Lf~De-APZA6SFj&cao{XfG*<{8oSakDfHP7! z$#I_FQDsM?nbQpmMmfm1jRg}18V(_M1Nm2m{02BMH|loV*+NLDXTR_#X7rHvUqzfx z;B)CIn91JZo|~1t+clc2*7n9HzKj<@K%Kg` z9sO+9e`X!FzU{qVD?D8vp6lK3&qI253}-6Bj z5En%NLmgILV;+UD)&;NYK0H~RVw8#YDyWk2mb}i*gEdnD!9i5j4$ccO5w!Nh_xsfU zLthOeIuau?OTIDU3R8Tct;#~OduD|jt3f{zw$9hB!E~C#A7C-zZewA-El~+sM#ep4@-~KR-f4-9N#M?|>VOT6ho@z;d>$N5 zUysV4wjl}+icF8y%bLS$s?9P08*y=+-yh+b%LXajP5CL3iS4<%?ma$_9J6No0AKHp ztdLeIO_spe^TpHZro5KEF1Oj{;`BfeCn010ZZb%OBEnN=ngK5W}ZX@UUVL2He<{#-C~lYQqkbNr7NOC zCTtWSG-_Kx!@rDxXZoi`BJMj99=vQC!388eEpsI%s_qxk&00lhvJNqQee$j#NK?5HIIplH^v#_UY~a81jSv9m(1f7^uG` z;tzsKwmAk8DrK{0Ff@?*5ZT9>wwclyDZ?OVV$ysh5t|lTJo%A=kx2m@cHm|@DLv7A06ZQas2oG^>7kedJS2auW}|hZ8*ewAc;Q(3d)ZY^4(tAXz@NUJSGGDaw9_sAKmIEooBly;)1Eqs zul3si*X4@HhCaOW-O!wv*M@q z38xt|1j)Teno9o}PP27B+2expnwy+njuCOcf1d=Xzt|hO8E4y?ypkHI*&L-)Huzvd zb{B(^l1{dn@s1ZnUPw>;M;*D!yue%&BC61g#T;az4%SXKQ6F;yC05q$%}A7Kgl1It ziI8c`B(X+AL_0N44-d;1q(_40tPSugZ2+e5zvG@nF8Qnt(IonpD7mklS8`uRqbd#< zdXK=};eBghesm2`V2Ncw-9HC*g&_&}gfXtd=nRY$83`2bYvSl$E3ZNI`@@Eh9!4jq zfy6`obsjn{@NB`g;z*kuJ5lBEMp;QPo{GI;s*j`9R`!LCXW4{6pO#TC4~{pNcgz4=6u1q0BKb zA$hOxNhm>|9Fk!o@gOW(Y?OC%S(8-OWT@;QB^FvUp}-Pj ztz-YeN0T0tIYn_t8OyoDqb8p!jX-;_eBRU<)fgb8CR1v;Hc8$qq0;q{dYm?s;F!WY zK@JrQ2BfBvixggnw3G!G^^$>gfGwvHhcbsE;qHKCfd3niTcJ$+4)SVF-vu*3rUK_n zLgQwerDMy*xBL6cDR;*ZDM&?J6t!$o9^J8$q*1fz*Z*1n18?1^!8O(!=lmTW~$n)TL1iAFoYMDAOiY?E4Ptvr(; zW0R|#(KAN|l0JU1luS80HEBa9dMUP4d+wZA9J5qQxiWX`$=;$I-~}i_xg!Dat=@w- zk)`xibG4ME#7hpcNc5Z%-e?lPVRDTY+f>q`jU?qtUN);F`YcY?<_c4KwSc1e3`HfF zcyO``;W@*oT>+<1dq#yN@SbM2^kbQ9J&(9}k)Y>Bd=Ps7yP8}7ibxuk)m^kC=V5x9 zJV!C*;cjFeZ*>DH!>oB!^UwJU?HUQG6|dHurDKPQCbaD2BQ0`Q=8bi4UB=GF!Fmew za(dmjoX5W$)73tGy`S&KOMaTVAN!pT)AFg{7QN05bUuK>zgo4RPVfW6qzr~IFKPT> zLl1-)bRFO9$IpG5uKx;=jBEk8r0oI}cl@(sbkJ%ki&YQ;QjltR>`PzGfK*sKckGnGt5 zdm}z{eq;x4kFp3BePBax+Rrt|AT#;I&}>YLfi_JtlHsn9oDHH-B*qbf*w5xA&l@BG zf91s?Gg{wwQ0lpbZ#Sz9#Iuz|Mu-$kOD@4+>e}?93T@!U4yT}yibdDqAZL${W=hLa zf>NEx!kV=mg~?Wx=q>dp18D5-ViTo7+^Q$CRGPi1K-|3?uokCP~lh4lbIb_@+wu0lDnS)szK4=Fzvb(i4b}EnaM(b|^ zC$jv4-&&F?H&Y&^oHj;5jgT`NuBKXlW9a-^@J?A_Lqlr17)~th7q2Q$t5_)xUiNRo zA~fWT+VUF0uVNVi3_USuZ+j zL>(KAzQZov+OBGQZQjsM`v}oTsdv3PvEdp7bZ3X?$xH~N>wv-)vO@w%W%BI=#2%gJ z2^l~AA(j_)ws4*m0tO|uk)Q7FAwi0+3AF)Rr{$ zenK4NH689&Ca91hX5oZd=1`L{#;LQ*$>6WS-XdxwYGHwQIHihk0oM4?aRJr_NC}GU z0G8ovDzuft!7Kb#7t~HUoSprz__iRU$S~?$qr@PdZ92{%Ud<^4(G9IiD)OtLUPx3^ zCoWG@28ucY{B^SM_~j{XCZa|Jla0bi!hnQ-hte2C*LtF*EX*46 z3cD?z$gS&Oj5iR|WIO3Ty4o9^Z=7xB4QNJd%g20FO*JZ!zLN)X&X_#uHN|g}LY@Nm zDGYrmWvFHfqGOaiP|!$!k810d0{FA`9ojU%jr#)pHcn!nLtKvf_VmE&i1JM`l{T2S z0G}4d??Og#ghL16mpHw?0jN*{*p^%}|KTqWvWZs~Ioc};k=ov}%>yD;aVWD&6%sw7 znOYT_6rw4_o8=Df*iAiyV>@Q{rv4$7&PK|RRMYsoOye{sJo!i;J}a}9)>}oT;7+V_ zW+}ZJ!(U9qSi{Fq{QOZRPoF^{sV}~mvkj?I@j^I}2dXi_wT>vLUFw<8+RNUoJP#n$xC4B|?iLsfo z=Xg%Ct(tr;Yz%V`Aav}p9=O(+H(`*dZSw` zOPWv`34RD2r8DJ)o@lH~$k_ts!fk`Gz^>!-Cy4&wozCAsMh0%6j3fA_)uneOtV3zJ zm{sd}`83Ye^_lfvP*s}f)qkBaGv*(i$M;L)Gp9#H)GXZ9@=Qd?1W>xV7}i5&K&V_Z zIyRge%89a?Y*@Ld9nh0Q62GYeMH_n7t+;$?%aIqN50psFOe=;=suGGdE-4Q`C@LlH z;z`Dzbpq;96jDrl>GC6u;TM)<0m~g{D-U^~gaS~7DSp6ia7o1BK=524(%~>l<>gP~ z@>Dg-NNj8flH;QkIVP62*Hmys&BF>!^5pw@bi%gM{pwL`S~h>wyy)(X0}CfMsjj-i zz&NbEYmEbLx?pC;3&5P2{07Giw%qwr@VSNjTvijEhyR0)c0-IMshZyYrFp5*fd)d% zjwk#7DB6)mgo6!sDe0KGFv3a=!?{+HTrIY?yc;^LI(#AOCESwm3Vhz7GsUlOad6o7 z?)-mtcf7Jr0}lRwK~hw^GoB7ZgY9O-sPJ7%?7cipU|P(BgDh+k<#=iH#P&A~6=sr* zf^&AUWyyaj^E>a80Gwp|WwFWsYPn7U?`6#ywzJWb!c_~pCkSvav=e8j-Qn~&c-Wyk zQ3wI1UvJ$o`uc_t@mhn7tjWW;X#R3LMIM^dK=q__#hPxADI)j>0#G?SA`|$pjgQl- z136v)GNh-iUrTZo_6x;O-oe5`%VYnmzz!h9rQg=%|3N#-_PkkKR5IfSh3g8=N`#}8!WS^dkO(L22 zuKxske|0E&h}8M7-z)B)0%;1;e+*yADpK;|zlN{-yJ^PFmAZn9RFDKolRZwXh!|nO zkf(U6hG;zc(=JR@Pje-qMC_(;iG?{6Wg0rd#rc|q0QnlC2v0@^r6zGSMrge#I00Y4 zs2GQg>6ojzNytY!zMU=KFK1uu?F$*_P?$~Umw3}>7-oXfq9i=|8;1!p2vEx}^b!yd zB|~sNeSIChp>_7TZ(VEMMyeK_2>|{HG4^)$p1NL_UwlpU1IF9)kiYQ(yy)qWR&Z`^IYc(tBnoQW_trA=DBb>};OS|y53 zHHzLZ{#$kb!Y7I2Gcf)f+fjA{!Gv=sc)*x)jD|gFwZHtzK)pAJ|W|SJ8V|5FvvFo-sK*=3fI)8e#2FD9R(lE z%w2X%lS|QoOK01jS^v%S`0rO|k;Q&JQlCFk3s31qq(;@A=S0AV5M+*pX~;Wwl`cDj zPow<&AEAXn4N8zHCm?t{yOxVlMJLXR)~J4xOW=K989pwyh-~&~R6@SJ6e^ zc!(r%X9*0B3}#Ob*+}R8m~CY!B7!}Z%e3)CuIMiE^ZI@_`LWhrYAwBc8 zr>_-V8TQ1SIvy44K#vhl?%0&3MeB`?bVUmDvMBaKgb{NqQ_%)70Wm3Q55E+MQN?{% zBM5PELY&`%G0I-yFZ=e8rm2_##n)HqE~Aru=Tbm>*#Kk3XZuP1EIf3 zaU}sC@?P$6pOB7^Fa_5iV4w3DN5t`FnR0?OPF>N`TOn4>Q+Nm7iURt3)PyQZ?mk0R zNlKg*DN}gn1#iS|QUDzQ#$ij)kz<6xn4V#{c@4- z+mO^B*N0_MvwZ^?a&}N#GA7M?g>a2x)~HIUo#(}xC~B!66xA|@rcl&guGLphhV|gz zV+mY#s&e9LA*rbQC;-&X7hE~$agYL9m0`%~yr9hysK+s5j(?5pl1NNxN~1^27|^6S z5KJjrTSASHN*P$ZrBcPub`^}CmUyTdJtp3W-r(a`ntaJjAsxCbsv22w&09Lw&5^HW zO~@ok!6YZXt#d!pZ2Z^;UjATD&62eUtmjdFGMoCPG8kKaHu;$Prr!LprROO!HLTsx z*odkP?9o^tt!j6-tXC02h5lsi&mwsWtrtQv_sO+dFmnx}21@6F?HS_dXQ?5dtq_z5 z*MsLVumKV9E!=t`b|C=>pD+&afuwWJZroSn+T+&ij*`^I!M1Zi4|B{neeU}ZPaP0l zuQqPzf__L*l9C8^oq@ie(()p*&HC9L-`J1aQ@D<|33epLhyZ&$@v#OBKa)JF&ns3l zH+~{98eCDVz@@Uh{^=Z1pWp#sTx^gCiYnb^`ZvT3Ejxy$AKfyG=L+PX12f(XeE$sN zRtx`Yl`0D%2_#?a__<)7Zb+SCL#w~i|OD(!;IV-QRM+Pm~|N-Wo$D@ftC<*A#xo>6ZpqUuNfw`*Z0bI$Lys;USY4Ru zUE9c7u!F=PjMD_@;kkWKSsqoGH}QXxALbrZd@*;l;F`FbzZ9W)3Y)UhX{OX9v4NFFLE+ZL-1Q5hqllQNvRrDOjT+iJ;KHXuo8nf z{JmYTNjN3Mz%;#+#@+i`F8_8dVBbDR#qZ2h9WOL#7@sl!xk=7278mQQfSC@ooH_8QBbqCZ?J1$fA->1haQ0I`={C$Xwj?<* zyexrl&d>ELozT868w9*WFQE$^u3A4ZNVq9+1uT^_H4cH z7up(p!5bH&^~!vqIm>bTz8=u8>brwM4`_JRy&+-8v^-O743AwLAfCp34r_B8u-EnT zmh3N^F+r}x>gy-raew4=;;Kkwpe*GO zEQxZ`;mJJyd|+rc&y0kzs{>AjRQPxRPM!a`!U;U)0>(kO6OT3i#c<0?BY^4L9>Jow zFGQ(r;W0*mo`&H7-Z7k4Ul6!=nO&v}PBN6wb@bXF)}J`dS2mI#J_Gn7;;O1UI%qrb zQBx3mPSutyXiPkX0?j`#6QhrbN-6m7lguq#ur&XUnXC5~XUw|2#3X-g*`1cpa6Wag zIShFF`R4bWAB8|uYvVgv+Wo>a#n@g85AlI4ITuz%8v%*ZW=@~X4@WhLP&7oY6XVo% zLZJl{HCGpzN{YDhNEzy_i8$V@iZc!Ax()YjTNf`9zm2-rBIXm%V!Cjb3nF!IBZM_r z!}U^StTME9m6c)1Qx4DKTEHUqosdu2**ir)pLD)H6Xn=ryVUYIZ167gdFg1POzk47 zC4(rtDgA4qU)qkaCNXEI}x{Vz_~G{kbp6vs@;93Gl7*;|4G zSlSV*H+)*^4K#sTEXS|Qz{=&0UInvpmO!oUP?6NVr?5ma|6N2X#nNH~9z0U?j;FU? zx+sl=v`n)SaL1{<5XsWS6{cHu)_2a*`m4@Jepg9++d&Z7W8H*}P{{a@H?xq&$r>kc z{_~GbC8??L?RN(H0-fwDF;}mk@ZMwvE5&Cl;pyii@@G|~6&oZ#WdcM^0gys7Y@Tam zztSQ4r&)5Ju{*+4zFs2RZ0Pgqz<5v^J!OLCpllJ8WA$LdnM@{;!BvPM_2xtHr5B51 zH7Jo9?!=@{t`im}E!RwpzAi&}5>d=e^Nbn7{hotIHwKvYCc(kIb6BlV?tEHT6dvA? z*!ZnoI?bV8lN0A=qRe!(M5(q&J6}kZBzYWWy-!rl%{-Uz3`_hIN|5(;#buI?sFfn! z)XZ)dcR-G0nQF?nCx4@XlOCZ|>u89^ywBm5!|fE0-WHr{(Gk;)2t)ks}Ia0tH3VjF6$1C3H_QCoEf zH63JX7)p<}V2muxl8Dj02n_sQmda^A=*@W-4%ZR)mmP6=N4ysX%_KEGkGEJ^O}I1W z&4^j%D&7L7m_XW{E4Vr?*`w(WfSXi{{OTnt)!88z{1G0qVwfMPKG_!FD`BSfRExq0 z_)M7O5Jz*uu2hiZiA?%chZ(JSV&Qz||1kHB&ylqMyKy!)c4KE^ZEV}NZQHhOXJgwo zH@0o_%kRy*4>CQ^=? zy?>OWV_!6#h!*S_;V;A|{CEIe`h|1?@#6n_7kJhMk78+GScO*oT_CS{uUhlN9Jt&wfRqmpMnV&h0xW*DJ{$WEyWI;td)yRrc=Ghc2G-yxn;jjg9QTf~v=ewtXndfr)@chnRX* zOd+3_jaY83OjIs!p^{X{=jXx7k{T0d9cFjwZf%Yzv~t-leL+G9Ke7ckErNuin`*fA z$Pk5ZMa-s%Fke9^G2%nKa3QbjK)idVJ(r(XEpzf%+;QeahbNT_ zl2#I~*LL)w8&LfL9Rn3j@ z@(g>waFz6@dSQHedtkhWgkz(|aWW(Nynh!Fpru$>rHEk+;2?80DWFeXkHCOF>_=As9bDb%taN>W# zQM_L0hn^(eBBFL&VwH&szv$wuW?;RW(}9J%xD|1UD9S;n{#cW>XowoyGHQyUHsI=V zU%{_x%5`M-w)Y1MovorpEjq?GWe&cE)sH}i>}(Z|!o zw?1w-_7?#&up7LEL(vaSo9gM463&!{CK+N#6Gj!G(z!>KXhlP39!4q#39~u`EY`{L zx_|3;5*^*J$ZF$Z7%Sg)hWeW-w0Lq_ss`ET-)N`7#do|ch`QhV`CN?D?=SC7AQS$2 za#TYONa=TCC}9CLf{NS6-~GAI`j)i1xq_{z=+2X5XCNq+xkydJ&YUz9j+*dqX&6>!K08fV%4UTr^SEg;RLk&49h=v-X+5ER_nj+DP4yGuWBx8>|=S_a}>G zx9fPR^!gpACuy-)yDP@S&C4Bp!bqf)KyEl@K!m-_VES*!x{wYNY<3LZ zV~b6=VLtRCBg`@CnK*I(G<05AQCs_b6pt5?HP$4f_YybjvOpCJn`K}LuZ<{q7QV8{d*xi^@>Ykm zXy3tu3z|J}HY-n?eHX1@`|b_Z%FEmSi2;=aa>EtoX6|wHg{2ofLK9)UdT07-AQ
h#!i6^=2a-m%dwtdl(9Hc3b~G7T$#ir2G@DD%3oXdeaI7dbJLSp zB)PD&qxOwcC1|P%PC3DGYfEw!(~8G`cBtd#aGiqzaj-~Q1W3&DoBZm9A28uuZ{?09 zAdFMACZ$GP!t-R>H1jDdhP%ZOH<`t|ztRbn=pHlXX=H`L!oVLUet!qdRO*1D#c8DA z!@S24Vc=NJld9UH7A_!lUWpf|CzW-_E`N(p}C|JcM7a^7Ct%8Riv} zepPbmBoi+D#2tRg_()Fjo1d_OhOhaGg-z`6D4UmFt#)EPU(nZ)N@Ur0>v~(hOVun* zDRrm1lD)5~Hkf6MRVTB+9XhNh!^k z8R3TOOEee*>1?J*91|?k*io1RrGMS22_8W2x<6P`UwcMIT2~^qTK=BrIbKp3c<2K% zlUrx{l<(A#uwJsJo|vx;b8l~yCD>ES-yTjhYtQo+ZBCXW`mp_YIm0L}GFxrw`Z}z! zI!AAWKOR{+sk!L0R)w>{?tn~VK7zfp7c1qOB-%#;sTAbv$Y>Sfw`nO;bdC>`WY zzI!?YR2v80&*;{SJ6RWir9CQ6HSQ{61pF4j#aWWFzE3IZ=6qR2|JtvzCDDI9}mCcvNe&zfW z#}dRKxNeQs>F>KC7sF1I|DL|Ql`464Uhj54Xu>qC47 zCJ&5d_RD==M8z%&Y+djj8Jj=RyE$aS$#pQLH@-^m0<#HwFF~G-R-7eFJ z<@2cdOVkY#b#rsW!^@t(JJLAY`PX`>o zC&Tq=1B7D>TED@*YaCwURo)~EX`YTl71Yn)7<5HhRsGF`opHraITw4nhH-dgNc>p2 z3)7q#;!dwdw9=M4g{PL0=q7x!*e9kORxtd7z8XsgCJkYr&M&AS2N5h7uJrcn1q(!J z*enH4)%z{t(dGYr7%jPdr45m8HbbDo>{8ZWugtI~8B)?;C%zD(erz3R(NsB33vw8Y zOh@hy1kD?ugF)^Wlfgl5MucOny;0`*1;Nzi?lCl49~le8+FzBtZe2oGmD|VjuNHkfcP` z{`=BUp24%6S{J}6?tfqB3XIp#$}z-M1E>$Bemi{6>-`3So*WVkwK#ZGtd4!hMAU;^GzG zhnHNhGk}cv#)8zb#uJw~dtmp;bgd2Yg!fhN?@uqoS5_eR(~T#|6mFKRUXCRIx`fnE zUMHGXMG*t7pQsf7oJ*am3{k|HZ+2B~4u8g?-bsiv^zw)^3zlW}y@{ayOC$-S!BelQ2urBM-V+;bl3l~ew)b%<5^B?3B7Tlf>Fq$X?^+=EZ zK_2EC|08h5we#CyaE{r2eRBu#1Cn2jH)kB=6B=wy`#wUOv6#^peMW=N5L+d9{|c`- zdoh-0OY@g`4gcruY5TDy!4f z&g-J{l26)MEK@Y1sg^ZL{fzXUO2w=VFO_>?)$j=0%m2urSLYS`6%-TCWL#&HT3Gvq zIwv2;33f`~>Ga<1re4zyj6BSUstc+K*}*5n8u0X^sdFNz4i|Yi;G5}{+;uG^j{aUQ zacrBk`wT0R*vcM?EDt%vWMq!6*f2$X5fW!b{o!Mwe@|9AgW~6@#N|r0h_wck1LGRg ziy`|(~j`woNW+ZXit%B?!Z@m_&LZz9B{j0+;E0fU9 zgP;IfNa+<3WAGmFz>Xcl{qt9HRB?2p0txB_MeSi?U<y%>y2FdoU;&3GjHdqNfL>*dbhoaqJp(o?Om`D z)F}v8Fnx1&QzV9Ip3hdNX@cCDaf}#YAy)9e7lH?x%x@0>YA(39FZ!c!f*t4AL}fLn zvD^cRa`qyE%=@tCtOBoGo#NO6>^)S|)#toP4JAM{WnNouZ^}HbMA$!+?BZ6RhI)Zl zMH>4iwYn0KAGwG57a>fUdR`9n3x-P*)HO8Wn@$A7^yTRshwTaYK>RfGxQ$aK%&KzY zd}3BqlcDc0>bD&b$}3ziSl_5XsEHY68*su2iXGL#!N}TtmX&0tnsF;GAY6cK<3!8%joc!ybzlRw}Xb}H&bwA%qi+Kqk zk$k2vvT`7&Saao0?jFxh=^6_@HQ84J-tt^JoXJsK=;2U)TZo@Y8;H`WU_HF2>TH%2 zsXMrm9A_^%!pJapXSlF{n`zjCFdeoz87;&)k}hsXnm>-aFjSXle|t&pV4dNj5o$$z)I`nnCILX%YnA4(xmHU4xlUu~Ina>P($ zouOYi(<(U&T#kT1dW~g>**;&IBAOGEak1iskqFoht$mC;b zIu}XiB9SVq%4>M^Dt$P+f1K=Y&b{0VN%K+=?a_Y3EmfNE^sa1aHGPd!AW`T9b zEKJS33Vd(m#nZ?72>K}xQiVwv22&YSszI&7+S_D8aEK3(Vq2hM zM2r#etKv;YvxAAJ)Wmz)2_Btvfbu0Od1Nr4LIiI$Qkr6<-?`2@NsSY37OjSr5*Ydl z+#!>ODI=VxrzgKd-5UmHs=-LIF#7EB+m3fr;j#^`PclHli?Yl=HQ5tF5RNGCZxnNL$q#t#wp& zN1X0BJknfsd*O#C zmu;}Wx3mHrmX51YAI^KZ8zV}}JL(aw?cZubd(*4OzF4r&0O%tVU$v8xOPFSHSLvBi zA7KKV+DR_8X9}Ae&J65Y%3dBN%^evwlfBV&^Pjydr1?*iz6e{bW>gGhgVM{E7t#wM zMt{)_J1O-@lhdb(Q#vJ8(c~fZ&|UUY9OIJO;>IAzw_VW5sZVZHZ;Rc4kET?jGpvS1 zByS4<;A3mBHX`z*a@x+BIXND2C5m@CteiGY8JA}201@lm5+02HY6yo-p{Hjz#0 zGVve=a>~uOVWi0QHv*d~D?S6ENpZRlu2|3)C1v_Ckq!lfkye*Jbf_d%vP8d7by7y@ z@YrR-#F9*auHl~V8vdc=avB2}ED}P?%J4E&?eF=WXH^KL_Pe;Kr|ory2M=2}c;>?n zOy=UeUZP+e>yh+jLGD!7jubA2RQ5)4>LLW5=Mg$Kt^}Mq`JZ;u_Z&%oJ=;SOVXPW# zT5?61H7vc1K<$eKM?jh^cYuw5L9u~MW3!g@THRabPNeu%kKw8 zToYdfrovJx#yiGgUPPGqO2{ISjFcvay#gnn45ZmZ8O-CeJp*W6ro@aDz;=BOV%E!# zQ&T0|;=)K1PCPDC8aittW3iGnz`9Q#zMWjVSTq)p?>e2b6|EvNCp=&Lc#)n}cdG`= z&p9Zi{e5EB-#g|I{9?2CEhtAEI(P=@BNWg<#qn@qXQC z0J;=KHbI19##Dptv?)ev0Q%o)ZFG`fu6ifeEI{pAWKEs3ziPIeT3~a(z*ZZ2?Gih@ z&h{_90H@Sz&mvQsM->Qs4}e; z-J~GB(icDHi4gN8yH^#8sSOnKJ1-jLqS$9Z-3wrv#mUU$wVu)_G(=D6_#K9$Yc5$Q z(JA+E>GNnUnlQKO@?A2N;JG6Nal{awRiw2O(qbYX+$HLvQ9DPQk+c|&Go!Ru^n^xR z>1sA8L8Of%<11hMtlEd7!rXR+jcRQ%bc<_@i>9$u;#$E|!hTGaa}ZIpgwWlosgMlC z?q5!4s+P4f9InWd6eO8gkRK%Z9ea7Ce@RK!#bMM}&zftu7F_?QM2dO+-x8@he#O;_ z4!;BV8WawY_KC}@nQSrYHp3Hg61dAEeTKHJ(3$CH}V*suU~>%Xex{BbMd&6cheYPi4OH*Z-4??pqKdiEsKF z1!So0lcK*aANNb4IDXBXf^AgQvM4!Tq^J=Oz#*0QX>NkUvcR!h>^>F?@T@TxAT?7V z3dkjt$i?bKK2|YyRhUE%#X2dh@cZ-giM%|!vhGw(GFFrigY4kt=;AKOi;7t{4k(i1 zcyBj=n}(nwxg(xJt}YqVSw4td@dhFV)rJmjpb{ukBY@mGO%-dc6AB%>^rTh4VG280 zVlZE^mYi}R|K1Z@_lw{sx!aTCi1^N39r~N5-*N0e^*YlA@*~>$ye%C9^ok)g>2NmHe zSB+t8Wd}Z)fMQ~3u(jZ#V`1gaT<5HaJ01UQ9s+*z7rrFCI@$5{ce(62yn~{)4C;YN z)P(Y{L@7-ga-c`*NHOK&?a8A+ra9cDgP#I9X9k}j7n-Ov(u3q0YnVm`^YrWVISp+g zo_!0o;S!fJsTLHvtwN5q}p=zLNL}W2@HE>awAvh1l^tnT{6ZzLGR9~ zM;Jke6ih5B`7`TalR`r8bs!EA8;sJnLUx5Sn)GUUttW#2T%eNIdQ;;1k;4dr(B$!A zQyTruc~@1V8FWBFm13f(dP$gGT(eHyYab9t4Q$zrP^IL0%Di3HU>j6Mx`J9vpc4jzxE9}|3AFUJ`j%fmLx2QQ5q-Zy*biEF;7-i$rKrAh(z(EQ>^z=1>k+wwEX zrEFT2+M&zz)BIriZ0}1D6glj+2>h%JR0_gN36z@= zWl3>fr#`Pu=L~>A$FEYkS_GLM9ywn8H4_zmvVidV@Q=X?*EturYfR^l%NNh%rc=8W z_BdYFv^O+tLe;WTKJi!z@ul?Pl}1s=YVCZJywj;pm7D-ERsN*LkD2gO28&HS?ITYP z8wMGB^d62M^6^$Hb+C`8;X>=ZQBj>f9-tan>`S1JWGYy+jd0gmY*yI%;dKL@5yZc% zF62>Wsi~nft{Im2-Bg>$i6i@pltI|*XAG#`UJs|7msU%$Aa%D)v!t#emPOOW*za&f z{XfVHRRrCahEb1oGqJZn;VVMUT~s}q`~MVlk*w0bodoCSSW9gZ!eWbSIgLk@u8*lM zhEqAzraiwk@`4ysw7k6ZOcH=!p)i1Y{}?(yN~Ks=ORlmt3F2VcnSz37=MFsleOm!% z>-h*Kdkp4?JCu9p;!DUP>Qj3G0Lbmd_xFLHmZj>>#Iz?Hd1h;3mF*#u*3N{OTM#2< z=F8~zgh26+Sl|bud13S!z8CP2jBYQ3jtjr3=GfQ~BXtY@sO?FoOdP~_tkHQcM9~tHOA^%Ar+^BvDi?=+yD9f>hxiCDx9Yc(FI1mm&l`ERx8FRkp&g^?PEmqRO zl3%vo_w`*qw=bgI&o=@Y>yb3SfpA_gT zvZrVAlQ?4Z5Tm;p5i-;X=)hp|w!z=V77jGT!T3z{%|5!jA!o6QN&dZNx8O9uS1VGl zvoKE@OIzRLwKh*@l)R`ks42P01jJuRt8toSuLs`#p2r`qtf~nqQaM>Ckmh)#8aS53 zD7f3C_w}Gc4DX~Gu&svE%_2Vy8OHe7n-B1&FtYE&Qm1)bow0*JCq?z(l!lSO9dP{~ zV7&K?kw5@`^MJ<6!NbXJlHPoyQtRS&?qkQm&$L^wnD%%A+=!Bq{j5ihZRbu@_J=0z zC_O8o+lC^!Lh$FWQX#O1J##mcGtAM>%IMe0Ze|ECD=>Gv!S&?BE74UYEVa0*px$+Q z(GFOuX@db;R>2QrgR=}5ORZq3-*ZBsM(aL~pA=Qf=`WZ0Yc{xDwZ}J4>BCzZ_s=XD zYg?L*ZyJ*qHq>n&7l)6{7TEP7B-o?xFE)6-vw8t{+{;leCE9jRoFty0rUT8v$YOp1lcEJBw2*SBud} zeIw*%WmZjXmpSMy!82>qHEd>x`D!2!lw#lc;?EBaJyh$>r@KH{mGAJDmwhxF!C&N@ zU^aYn;w&RH!@u1n(Kr52zIpZ3cG_o(1W=xIm{9oQ{hOyFBOz z-}2>x3{o_-Tp62sk1^I-|Ggc1H4AOumJb#rsvOBGvo zu@XfBxqA-g(vup8>r0#D3xAT2XkYEGbzeqYX0C&qaMM!DwS^Y+<7p4r8s7NY?vndL z;_e$D?U3qRmv&rH4AUS&*%#^3{&sX2SE^%FA1_fpIrHzAwhQTY764vS!R5P#RKxk|JS#3fVT=#^1a0Qsp1&+< zWgkgD1p`9$A-{&)TyJ7mikz=c7|DsX%~Lj?j!f{>aDeroKp8`6xZvN=(lIWkJW4|3 z9h?qB3^d)@7+g;#O-wvNAO@ZrhC5g8m6z?8+~F-`R2|}zin|?kwHdTrbU)_C*Y={- zw)|9zDymybpuQ5`#EW+3OK4oY9rW1j#`IG-a&4Cw7V8P}4FWl9vgZ_!+t0At+s|$I z^RQ;Ne6_dUmR+c+*Po66>8@(){+;KPk9BkBMwQ0fQrgSv;++g)nMGLzaSK&LD72QI zLI)%$K}xEx{u?aw#@-k~a+r%fl4XC($r1!1Pr&r2`&gF4$GR4bUT7!B(aPqgxQp74~JW+kZvIz)+2{3R;-Xi)W2M(Q-KxDfGIoLbZ(wH<4PDPKk(| zd_195dog$VD4cwVU#Yl%`9FA{$LO7_=KOhkR~)`skwH`_I;IrU1iZTFSl$q>jV%7{ zTX47gT`zgVQ8YlgSgT)Ap#0lakmU0jwg$W^(WR(^OibMEC`Nv>eu4c!t7XB03m!QS z)D^TAGz5MX2q!i$`WYQ_qpp8j{^S#G$IC?FvpMm$I?<=-omTZlhTBf)?JIPhIrK*z zF_3SBCO8l|K%2oB9B6@twJDi79L7TL2d|MB*;BZx0(u* z_Nid2enq%i_=13Bvmxe+RY5DT_Bad!Q<#kxjuPZeKpm`l zCRZrBlcZ24eO{hb`Hn?MSn7>+8b;N7zN{*g8JAiXTVUQ-FU? zl~=W_et!Z>z0-D2ulkBX)b9oKuG4=nRCf4YKyHo8G0CszUmbM=lsH|frH;>Y?SLCn zby=2>^O@(k0My zgy%V9KpMXSjOu~_I?%hLr@QMT3T8kjOii}g&jv&d>S(_BC)f@jvU^6A^i2~+rIIdk zy27%{BxP-M_+a7``tg3PYId7)NR2<4*2X_Pz)0KFiV=Gne*v`vly#nO&aHZ*MsMKZBGeT6<)DwGMu<6jD<12#pz zq(#%@xhJz}>-Gjx<0A|z;5KbMpB0EDhixk90!!x268^08ZN(!Bxd|Xyg+yIN9eKQD zwR!i|oaW^8R{MQ;^{KU;+S!6+H}Z|Tu31tB=u#ZziW2;?7D;0Odr(OE^Qglp;x+8E zcBH27A1$;;TXzC8Dnh^hy@=176F#1e_*PhOY77snV_Q3a{prF0RyV})r0+%^-glTx z?5~d>9pT`y-5nq=kFP3rBwNhHtlwy(R1$;ec#~e)m6M`2B}jFwai<2)=rz9PZ@g$9 z!LJ*rxdFm?P)9Nj|7WRh+@MZW7A#xVM(WGyqoYHo4q_YG1=dEeKcwgLbOy6bN6hOE7)?ly(bZ#f|C zZV>0-&E50spHum37@&_Tq`A%4xT=#lH5C7%i`L+^Rc8(1C99un))omL7R}D*$+Ptu zNz5=FMWttwTgVPv>tshGf z)?3Tz48R%@m%YZ#t&mh7$(k)a4jR>!$+bp}e=4b?Ka71!TBdSZ^`qFP0i_En^ztQc zpQ=|@XevQNY`-OD3Ln6hI}U|!`K?A%8N5mDX{_2OS}7|u*HCqS5Z06TDTB3z`;stl zNU6RR4U*>6ZFyQJE9;wuQcfkMP3)rgk0P~lxg(F>}w7ril% zW-yZTaV?o#i-c}O@!7V(kLp}Vzp4lcV7GU506;R$vLYVU@)g&`RsSKRfI#!AiFcY# zJ-4}Y17t`TPW;cs|DcQpLfJyf3I74ySGopcEq#8)423`{kX*CZyr51w^T1d1<&mA5 z-wy(Ye)#uNhC&2{Kpuho!+hx1T-&d>(GH%sP&l>s{U06mzv}3cCbIs4Da`5L)t5xG zAF!y?fl14GWr=@BJ8bGyfbU~#@-sMR!1zW&&{bJ9&#i!_5wwoCw_?Fd5wx*AUnh)D z2Q!*C%B^MI5j*fHha(!@rgoC(&%#m>5 z8>~+T<<|~J|%r$sl-H8orC~x!1l+oMzt}!Y*OJ%+C`;wo@C5LsZ;?c}X949>jKa78W z$WwT{e_gqI+OkX9+ceMCXKpww(a}zJ66G15xi#4!j}ns&nen859VglH(X*U(6UP~l zcP3F;<=RTjQ=NS7*$;J0msinm-;>Tr1UCV#WpD8z=4oO8JQXW z2)9q^6dWyWF(q{Ux$Y-O>7qc}Xu(wWyN*=t9VjSvNPMdOEBEQ)caa|@sf zPg~Zp;Mp@GSvqJAghW3+KfKJ39Yr1q{o_`J%D8b7U%Wb8bIVn6M6blTifzB9RX<*My5N?|m8e()l`A{~c zt(rz%BD9$c%h+Sq0SJ@!TzI@dF zPrZ)0GzWKT@%V8EvO#2LA}gdISWr^Oc@eSAxt7K#PIvPD@F(W3c|-kkBXPHkm6*x1 zhT6prhI}Q*JAWcb&fKXYc(#L}S^nH=!n(Kxt)W-3I)@(-7?q@HXb6(ZX3}YP5tqr| z4{a{A5Bm%mDDX?5xBuC%bi%;;HLf53k zcd*@lC=~(ha)^Yca#(6zVynwn_|<#7;Xf`qL>bn% zV})Zy^6&dcAzE$)wqFU{6|X=GxJw&}jT=GmQGsrVfkH#^M_WC>c_-!@jp)+aMR#fI z{7D^zs~K#2;!F23<&+qPF=CV*`msWBhCC=q>QKc1O;<|L0~vZGS9EP{zfxx!%1Leu zNkW^w?*(%~rrHa`peaP{DNA!>A71#t@mj&Ma>1eTN9MAJkRV~w-t#vOl-hGhJ##I; z+~WHXaDVU|Tm7|%i^e-Af9KtST-oRP`R@A-lVtJxHrk7M14BTO81gIG;R^u_I-tp< z!>Hf)lHJM)FXA!5nZp(jk+J`VMc=nNxyCTSB!|;ByFMxW_cU1eHnZLRBmjg1shX;Z zvhAS>X!hdQ=(z)2bD6|%`TisLk} z-3D<~X9B?!uhL7bZ3|=tN<1{zFL*~cI#~j3)2BzI^c4A7TDsrgNsjsZouxO63Tx>| zjj5m`4^V_C{6Q8|-8{stYaTR)yx1^E5mN7Kr~?X6R&G#EI;t{W)B?GYQZzuOewrOL zxIB4mer!tzz7Pa|4FwTs=4sv#N@DhRF$x|(mu>GmIO zg0;OqOodA(&q`U?AbrDxCqvmt9q&9_rW#w;>wdnjQ|}M7ieWL}!pG|Fa3p`!A$pS~ z3P#Te_`ccQ*J19vcqcP8-8`~jh)MX*io2p|q|~Qs3b0pm(DxfiSMWuhS|16wpS8rJ zSk)9$7*PF{uG+*`7P6UdKsM^RDgy@UypeJbs03;Ex^+xmE7zC@ZgGy?sjE6$|n6w4{ps@VmJAPdxbZUs#uWt6&dE7Y#qf#)H4vk&MEpqLoz)GiT} zGz*^lf;cJQhu(ph9@{#YPY6Gp{BoBRb$6#eS5E7U1OjLP!vlUVW*4i~ybH(3nWp*G zSMUZBejb}ao?R}P*Ri1AoebivjyFcb0QMw_a788)plC6aDvZ@w|cZpgn3uunFk$y1M0>^Pcq@uz`qzsjy zUewDT^2p<>i=k#j7@wj1$o+Cd8JuC#yTR0$hyGql$20!3nDUGYVYmN-PsQ)5*iu6Y z^^D-ycu4Jo10{}rNC#b9-z-wy@T;RxS`0m~TD14!O5bqUIf(IthZg#C&4*Dn94++n zJ-kah5QXff>z7Rsp`9`DIWKz}pA zj!ly+H1G?6ZKqH8RSFdvJKMl2a_;4oXBak!0w5OtX2HYEPZ1<} zb`84$@)$BweRw~$>{zn&dQF8@>|q2+&*~S3ek%w(zzJp{Tgxq3q zh?X$|})SM_?|Q=sny$ z>aouDhaPgBzPMs)?yf(6jM_|uI;QS`{QDLmj#JvhZ`9cBEb@DOv1yH=&3b&&ol{R`Wf}u(JK9YzPtC@F12^MUt96Z z9#ZO3zLOT#YXHS$Sis&HA>>jMIA~FeiCM-jjGs{iRYRW~1&pSqO**f4(e#If-TSf- z=!-N{Cx(6jhkk)Rt-Ho9%+ADzg#@-q?be*kea2PN@6bLjB2HuEmfYAwlqw-j2Qd(e zlfq2b1WK59v;xMexD)hzL=Hg{tJD7=zB~b5|3!SgWJ=It{ul8zLJ4*8pTrmI{@>%6 zR%{}%-r_5}X%J-})?a-O)-d<51>o*$Yt*`-#@&)VE8`(BUBDjj&}Krs5=*t477cn<`vhZ1@A>;^1{NQ%hvS8fstVLbm8WIuKwQLB2S!d^vFw~RDR}&*JF2Lc zQ}Bo#oiMzzK4w8t)G%%swI_niqGGY*>M@}vXP0yVIs#LjKGpV736XOa9=}s`1gq1V z;=PevT+CA^nK|3#SkOO=Sx)0ew1#nKYegX7u}Kb>_U0qL3F;!}6Z*6t74?}nC`$;p z7(j)=nl`)zy1t#~bHK_NkD}$ND}Jp9&GNdS&n)C%dxiLCvSoBwLZ2BlUC1@vJ@GA-zXiIJeNFewI#uJnxpC+zH>W;z^?YL+V=5JQq(r!54ht8g_>RqahRtMw{UJ~o+MUCfOi}JyW=-s6>~BaPsfq3 zsbai1hQ8NE_`8;Sw+LHZr7vn>gK+xhc#g&0Iyxvb-`kMDB-~q(Ty`NQ9vTO))hbN3@|>5kuVOx#E~-g*s=>gWH1k#qdRKwbP7`2PX$N`$Mt7 z#@33c>x&GBNP#VUk9xSj)5=q$$fY87^Jd>R^X-5;MLk)$*AX`fZ!&yqqOU#~lupfv zrfeqd?_qr7pB)aP z$wiZJ(^QFg0M|o+of!CZ1#?P;5**&0G4@&cKQ~jrxua%r1)mOuDoTb7t{fSxlvrt=t)~iX92kxKUTDC5A(bsCWm7lIZNM8{dkxwY?O@?y3(6@` zk%wgXegHt&b>EeM-C}_E(eM+sgPtAy2xiq%!0{D$bsp7$y})yA-2B=vDp!-(v234& zfnA_m6nZv562*%_y(#}BCzi!CO$=C#DN3bBY@y!4w#0(21+nD6n!V2O4G5ksSUa)Y z_sLWlII;A&;QNFHNwQ8V$!c@lDgR2o3!DoMl2pwuZ}Jc-YP}NRxABvdSC+2om*)$q z9937;H&Y5rN#C<5V-VV^zhDp)zSrJ9SmY zu_|JA&7ra5**|;nVn=!x8u1QsWB-am!^Gkd2kkBrwqNsN4BMoNdaGjotF)zf1twHZ z?d;aP{8F8)@cf#v_Kx=6wVL5W9p0RH&9SA3`dM}T^xbK5^)C19k<3o?*(D;nuWtjk zsTq*Bm$6g`=iP(q37J~c*m3+mWqiS&4VUFpKI0ZNMd|1b%-e^xXNBN7j=&(^*s96k)5Qfs7>;^1Su(Va=<<(IGH{7E=&i)rxCa>Rb)t#EK`CZ1A5$O-weFmHpMi zvVpkfZn^deAHq&^8a%wXw1#iy&yuugV&Y zvz$()xrTpsSENj%IgwulM<`%i6G4K{&8qp8WKQ9Di1;sMm5x= z*}vMTAVHYjvNo+ueuC>y4Z4o0Vz~la#bRb^%qW82R+aK>Le|^7;hl13bm;+MEfU?vl3MN=uIGKcKc z7^q@IEhc#CUuRfXen&A&EUEgsszPugxECl3!tCFGMsOp|4Ja zyn-KoCnBCF*5LyCWn!|z1(e|~jMioLsMI^q>9bDdbIN1ruI5BKrE1D>5ky)*Lb{cH zuR&Ex53OYa)bLFd1S!H2f^e5I^lvz8)l*JoUM{^CPw8t{-(u|K;Uf#i#{U3Q(oShL zi>UI8Pbk|lSWK5}|1x4aBY0*E9!Tdq)MXfJpOV&(ETLwb`(jC6n;ov2bPiqWmXD>$ zi|(o>omVZ&*q?<)EtT18tBx9L*5k+D!tR=tRh`epV)O=syYqOBdeS3|eL`ro6h@5w zKGlk(3Liz8cb@xe3r||1sfR0FnqX}KczuCc`)$IW@Y6eRQ?X>vV!lVsiq$2sL^p{D zP5s7yQm)CqmWTQl0#JIL;Zf}AM==k#e`Z)qjG@vSvj0(ejcR2z7>Xlv!ZIb+(5HYx z|BG)ujUbm+DI5>4A^T6h^^&j*t>$9E{_m0A5w#$Jvpy)R?e>q+4f+?S~pY;oiL=YoU3D$?q#_@6^XYj?y)=c>cXGLF*|dyjBol5Q1RM7 z<=cipbUtGs1JasE$04=U8-l3KhXA65fdO=@N_KEqVaV~Xp4-yTUe1FJx8wdFf-bXp zh1se$x8^n6vV*KBrM_TW6_x0D9)f_yVo(s3BuT-nFmC>Og*GaXK9)WYqlq!U!1OxlctbvclbB}}DxyxDL>HuE%JB87pQj$*V zsfuyA_~N_MV8GTr7wzV(#LJ<5|5jCTWg!-~&XZ?|o{l3TVTeOJaBsKRy2|Ax-Dtui zS$%ttq#e+lHarA_AE{S*mR^gg!l*3LFPoL$-tmT)+;-kd7xSx$>`WMZ$U!4fYQD-f zxr#h9mf2J({(H`>x0f1*#V`f>w~pP zR`jI&a(&Xty~_#9J2DEv@}&{dP)e3^&Dr-!dsFm7I5NrM@%zYWWor6lcmg%dG02*X zAt#zRXv0GA>Na&f2gfowHqGFL~B?}on5BQa$UHQ{?OwAWDZX|DOdgvZ^Y)kId%;x_Tc9ubLeCxW$ zWv~H;0Kql52M_KVAi>?;-QC^Y14(cV&fpfDzyQHXfMCG~7?{KV?6dFPx9U{g4_&pY zYjyYf(A~A3cm1CA9Kd~}(zG;m!wQjS=JxESo1#KoX0ppm-v6r->P4g23`GRcB)yIF zfiuiz9NnYNcs!mCUB6@Xkb#K2%lww-*}Dn|nd1Hc*>$?5_;vdLZVHf@sNEdHzv46)cO0eP(h-SErbG1uQXMmN}cwOK@>nj_u|rC$k9u>d%YkV{`IF+ z8+d?4|9YK?8=xtVjxyww&3(A48dzZI3uw|~8D>1d091Estq4AF`;=(oR_n3lBQCFK zh5uL-hvLNR4gYcx@afvXnL|8e#VJ!<(bz8VcBch(V;sUtplh$LGQ~U1cTaou zNtHLl3h27%Fcfz&)O)kFR*?~pRy|_n7`u_em)6LHT^5-_w`Nx-x1UaLe^Y(f?l3*h zuu1YqARXU`R_}%7aD`eECE_U}^-#cX*}yZRE5z;Uy~UxHlKkYcQ3-S5iS4KEg~$W+ z*ex4r6EikuiigGM6z3oM8<`)B3ifRD&qcS$d~H#f`y`REqM*_6FMsQ6`J;T8dye^i zi3!~Xy3DSCKf(hCyq|8P!-tCb$#1v0(wDPNIsrIZ0Go0`&^8r=*l#{wb+BUJ#wNcbK$hAHpDuFt;lF0W!EyI@^W5xTx_n&$ zUZ?2{6JI&^S+q-3+@)_`xyS!77*G3XGXAkvEbn46%4b?4|N3k9n*c8d zA?!GLUq~9XX0ojHv7_9JxvVvDjwn0(*Lz+4VtF$=fU@$}AW^nc=vA520w|r4o`qhO z#_m_^L2RX}8}fRWNcyPQ^~*|x7R{%q^)M&?_PI2&IYsFQxUAU*!|Pd$ni{k_4K5=> zl{@6{fkp69_Nz|;FzL`E*#~8B+(x-61N9m7H4HR8aG0`F1!^M4Dgp(0R2wd6gK&V~TLWyVWG`3M)YR8#UMnr%7v6fzPd&*43ySk)k5c#Xf zLm5nsXc}2#z>L0wy%9P2^H~ygvj?6oUgT}cPks@NpE?fK7Lm;ZZa9+o2kAum0M|F| z+9S5#h~x}ZG)PWcz_uou!KAHGeUqrupCTP{R_dC$GOq5hleL$&c#E^FLeq$29DbV^ zdzX?NN(5I5UUU@RzrQd9?+hlxM%B3qw8UUKO&->E{LKVc{(`TV4&sm_rGsJyt^66B`<3I)GlKKr~V~+6D7ZgeY!t9pgxKf+2q%Nb=xI zk+_1V%#MyR;4#g!x1et?=5ema+CAY7PBg)RqDA*Iz0Q~S0rLTm%RkrfSCCSM=GFr@ zkmSytupGS%O4oz0+6fYNqT1R}1$S(T!T{9wyS%u7WWqMsP|?{k^L3d=I`%U{b;NN5qYwzBqm?Am2+X%X^gI! z@<|cgV)j>U4IaiT`(gCX zV`tr2TWWH^{4tnendY`gdDDTQ{9ngY5h4y@>W4)2Xg^g6`|jRNHuP+uu6f^aN)-bq zrY`T2WbZ!&MGmlaTfaV2KzZ50>owV*R=O9)$W?)e3j84S1`)fVh=c-^Jw@bQ=ng7S ztw6=zQ^g{2+2lF1>-plU&gisKYq6}<)Ys8v{0%knTR~R5ea{Z6M!X`l)3bg9cdbXn zREUh$tAOmhi+O24BMl}cuz1JDLk;20dsaW(M$6aOud@Blem0@bIcx&`c)yxuO+F+j z+9lX=t(GiKc^Bq(G>!gI6!ZY};IaF|=d_*}C-qXWnY?LPk!#z$T{r&rGWjBNY3~Y8 z>jGXS&c8CD3}sJ|Xoh-x zTlpMH+FX)p4hB3h?HFO6qwc&*YtFphr0#vBR1LJbl@@=!nt6IkD2_BV!2u;h@s6$E zG_vRagt&U@IlAYzUWl!skoKkJ$pb`tIbv7Pw=QXXpcw!cI7vppoF>IeLVkB7{RTiz zrC+&t{JKFTdR_?X2S4ly@hmMV3=&)vFQj^^Jue9qTi^_a%`pm8DC0=P^mQ-crhgrP zKi|Pa`;x%b9dQXEg$Rs>*ff&cU?1c|Rb6zEzfVDeh5Ue`a5>*<*jd z@=rm0ahWAcYab1_5<)gjwoE3h_O)K&dw!>uqZOuB=LgJqtdCqCA1jZnRnw0XUNnrR z&!MP!hXPBkLzZoW@x#}*oc7N9?+vxsHpnZh^ZzM?i}davUhiDU!lBotCGo^CTb@?K zleV(Gmyk-PTNUBh4D==$+4PLb3r@y#P-UJDHJ;<&mH166`NJF;XS{igk5;~NY5zyM z@ScwS4rr2lkuHS_EE|fy1wT2sA#Ythl6)b zYb@B4+INR_-O_su9@k%x(XF;1($`$S3f@JtJ)R zS$B&qI2VL`JN7ST6o#Nz)@-RjYCIGD(`xg$L z+XRD8%EJQRlC-gc5Cnu7kni|-hd+=q5^nti1^c-fVvSF1(|qboFGUG z#QNmVaKorGbk8d1D#|+lg{t-6glKD~Am5>%Sk#f_%qOdVX$soMESaGH&=fjkm-gUX z!`QTc2+_%^g#Rd!f2hUz;@OgeM;wTJOAgY;O|eRV{2SXCc{`W>4|##%qj3^%)cu%8 z=g;_cq*0yOf#_x)>5#^Z0!z1qu#*Sxv2(^gjRrNdpHv!2 zlxlcVDHfv~A%G31GErRHxFhIfOV+{*x97DHzMoU%Ph+etM~-nPrO^ClyAE7;tZfX2 zqK5CZ+|nful|-gVkY^}53j0i1&6g$?hQyU?iV5rCz9l{_p|S1XLf_VN*2pyR{_1Ld zrRR0x6UV&0niYrFhQK~kQHE0mL%cSOXt~=^Q#C>UH>y<*bEDZ&J#t3X!F(Pmk&ngn zqc+SCjlIIbMRWDcy3<#X&3O2hlMd`XK&bkmcHi$?H@j)iSsOx4?|Sf~zjS8g*Z6(* zKBYY`TAX?;31}sZ2~I78YtxE6sI7Q|wL(ROBn$XU>%+vYW0w|lKORjOA4{A1MwZ$U zPt!83vJRwo%@XJMSnCmMY8P}i(sT}HUgeQg6v0ZB2iK%0!T^>FuRAV>WTbWmb6dM+ zY3_y&b&Jxuo~Q)4mMq9`Hm0`FkqL!nN*@mdPTE3$SRaK14|%rvaH1pMo(nA5giL<6 zq{J2e_4)YwROB2mpyI)bpyZ?0)P7h_9E@x0KE3B0dJ^_xVjBm9U7sC)LT454qkdF9JRaiZ^0g=?ejlkM3r4jNv~$eagwQ%^fb zrZi+vIL;OPgcBA9iwSiiya^zFS|Yk;xQWNADx=1f> zINhJ!B@$i8v+;|{@gjZ6$;Pbw#YexFDD_Dfa(O(+O4{DjnFwk2wO2Bm(vrXGX(Nkbf0II+YDEW( zVkchEj%Ne4*EU+UsesyHClk_UQvd^P$mKipDWTNILkN#JLS^HxyvAR3KSr{Nd~3mb zAJJ1*{WT$Om&s#~TP!^pp=0Y)u@tQRmPaI6w}Vtsh|JguW)kXHl2lR8x!SYgEKnHf zVkA#@omd#tw(pM#X~Z#rpf=bIUvODdh6_!P6@vv~32)gV?(>M-2o3@@;za?8w4u7Q zFRf&3XQ4FF zUi6cex)#5mpLx=&aZ->|eZHckZQ;_FU@@+QCG-k$-9-BTc9zE-Yq8h#Gy<<6jstJ? z6MK&UrieD^_p0WYKeV^K5wuZV`>%z!jQzW2+5)tXQF4PxnrHCn0I_>1lI{be&UIdL z&OwTYTXDc@3aHl;Am+Y?Cn^@T3StRxaFA|aZWf&Ytoec&K9c;ZrKgfVQ=L3mbTt{N zZmuyuF?D0Hsnggt%VKGX>+kmp31CM{|6dM2~NFCAo7F6IyTrstA5E6K3E{# zraXvKP!xsFRzKR5sJ~=^4pd)NL{tpDOhkKF?;p=+)zoH!g~gsyqgncXtpdFwrSt~0 zHv!ovwHJzjR81_?rRL0b?wxRICuVG$UUOfQ=k(zP7FET)3huNg7TdCUcQ&3R!lf{rvIx~W4` zq-Rfuqt#J`(+Whj!XjF4DF{;g3}2GqMrSW6NjCN;5RJ8YMtwsdj~CGEp=<6w7fK7< zS1n)p6PoDn8;d(u6?fJFTgDoJ1zY4HUtWa%OGqPQ&TE8YH^J~+zrdg9fSuqurZm9L zjDI=s%xbBvnBMeLax}&fph^F?Re3TZ6OtZzn5nvEXERXDF8n>o-OC^cJBUW>xSY9W zJAr@Dk0EqNc|Zj)!#a$kZC^z}b5nNC;+|e8U5l>x97mKd9JEy$79|xp1m8DaKhDV) zDAnNF{>85kZd4}PIA3(c(4Rg~>>?x6QZHsZ>`<(D^vS4!+2>>PtbJ}FRp608 zA~|LI9bRPz?XMI+ulCfnM*Qrb$Z8SFgSK?7ONyL5OyU~&kGNjJ)Qx!|QM^S#R7uR} z@i^~egOv5qtlO~v+l#}7+>m7@p`x9Q_;-o#gB9uf?6AR#?1^T?{z;xvjLI~LaJf4U&L>P$(1bg`PT|K?{|ZNFTM@Vf;^RC4d-OXX(={ZlstQe)lfv%fCRpR2er#SQ^Y_#ek2b`t+3xq*MIJa z*ikb8mYPvPi!ji|_WkiOA&iHVDMM0;YML?PWjqLUpU3#{G9T!8@j$#xSD`$3^eH@e zna$#FufRL=6Z1-W=(jYIuMBzzVtENq@MyyaB?Ij=$xayf_5S`of20nj8?HbJ^;(SH z#6YyC16$#SHiRB%Nx+i$As(@_!DS>Wt+Xhg@n$$6w+RS&z<5JbsYN;PT_GlQUxGwz zsVta81hJL(aITU8kL*;b1e8$xs?bR%A3^`}kHKQzm4&9gQi=)xUODVBH`j z4d0uu8S;d@Oo8KN9;#?6Y}(3!_kBBbYfLmz@mFu`SY=#hhSyr15G(^S0VLja<0T4<~un{Bbp$J(*H9a^Pt$MUEeJ3H|@0 zP|(H7Du8L6YU|e-?v{c!YaCls*})&C9RF)kSD zTC)uUB_&x@&BpO}31jWws_hhgM_Mf!K{^_>Z@1Q3nLn$oH6zBs0C1<D@ zmAvK5=GsJv=KfX-IpOU~=vyF#aPT<|ds>T^^Ar#5GzHljw=f*&HQx{%+^1#4SBk$y zD7ZBbuyL_*Fv-2pOXEBJciA9dx`)q(j|ixrZ!z6R8r@kLa0E=o(kHB8K&yEfwQ{ZLO*f-^{IX?abQSPwTJjlM= z7FS(*!>m7j9uyoW2YJMgqIX+dv+H+u$(rI_Jp_c*@s!Z-JcidSwI4DQMJgC)_q(u7 zR$tL!TjLYGn8-A&(ffIINz;y2OBhhniO5W@9hH*KbbF$o;LM4 z&ulmDPad@j9_-Fx*U)R^Imz#s&}QUUz7rElDb`5&jH z02SgHF~{Olqm6J{)UExecPDzWigsNz@nI@-nRmngS_>q#=;@5>y^er1)K6l2bo-cR ziv16Vwn&jt05APz-he}4V`?V#KOou&U%Ku~*q~z$yW>`aggpt;s0pjRc$Yb>`A=kd zlx^hKNff#`2l7DNqJ%h`WfeL1KuH)AC|M9c{exxg2O|s6_W3au!*`)#MH+6}&r~{- zBKmKk>6oOd{M&D(o-;V$Ijc*-k_vua^I-C znI2shIEQqph}%~tpiOq%m62`y&O7Jmcrx~Q?3Ml2zwz_gyX|S=T))b>mET~Uy<+{3 z*NY{>db={u9Eks?dqi!YLV;mV+(U@wc7#O%gVc5}<%XhrE(Mfv@oy!ZsQ;pbXU9eV z$cU9XVKFLKu=YSeCM3@HckO@IfhLaLBjn!qKI%{>(8+%PtFaU#zSOIhqh27{20j^4 z!2CLa2?oOM|3i-D8&2%Vf5kW?7VLQC1{NI6i#4`9TT^9hA1ys8ad3cF6~B3)*Oz&t z@fvtU@8Y?5SO53(yRdqXG5)R%XCE7f1o--?_ee-mUGas>DF5RWQ&B1GOn=jRKaU@z<@;BcY7GM+Gi`%pUosCF|Id9GMT?`4UA4^L0=@g%E zuD-flj48Q&Ko`drGa)ZJ&kn{BJ2f(dd3py!LyUt(|CCTsVZYE$0gnO1@(~q_L+UnT z)lVBrlhrL(!PS@iPkZ7RZiZV@=PW`T9BN|46qI%sz`sP$U&N)ouxm5dK<*Afv8Qgd z&0gf!{YJ>Jhhr1OwzQlJKe62$bSk`6rY4`>C25o{O4@kQ_J}?UI#KaIR7JNbvufxu z0pcdZ-77?|7h<67^&p2HzeVN&M!SeZRvtuF_(^-S|FcUO z_~H%6<)P7CC^IZ1z@~=vFMOq-!3P8niLD-on*meR+A+zkxBBgfT^lDTKJeC06Q=jC zhq*|VW6LaQscKDu&u@bBtg=9;aem68ZTC9*}zQ2@vsFDk0bE3~Hoc5vQ*oG4Px zSxOOMdD`=wd;e`3D48!6{4;Qjf{bwa{-C6^RPbkR--*|x&?WlWADeR^VFwO2USdp|ijyExrif1!So)>h;ZVxAP0c8YsXUGse~ z@~Ds@ULJSHur!a_vaLz4)_@6L68Sod?pK9lQLEOUf=y#&#LcRm%=mqrS!soL+(T!itn64>F-A`-5mznK`vzEgn4oLDAs%U6_f4hoQcyyq-9C9mu$1GlUo96OGAaOTeol z*`|zON}RmERZyMDy9HI-FHZ^H*Jx%vQvW)e{jl47QbOJ4BN@{rSzUUaeVJxUa*%EV z{8e*p=8%;6P3&Eckk~!V#)}WmrE8FpI8y9Jb!!#%DMB3!DwA%^CF?Y5{GmVNsGGwQ z>4Y<1FhL)XG+OeeAaSk1yh>u+c?0&R%0NMBhCSq(y%npg!K5E;8&xopqY44vbO&gp zg+S+i1AprG7L23;M>38`4}YQn?J=RElnBL2W{FZ)15+LYRMfO{?3*pYCZ?772p*mr>^XG zvivH#KR;FzOCE4oA6qG3^WM`CsWL>Xd?#lDEw6Tt@%0Sf25R>aVY!s_0aI$d>L} zZwW#E{v9p~ePQpbTMZl94Q1`JvbSnE#Q{4+*QQyXVqB+X`t^i~5clIoxaAjjH*SN~ z3eDfmrA+j~P(U8xqGUJ2M9Lf%qfw?f$wU5C*0tiNYQRDna?-j%bSmVdxFLmY7jwXq z{4WXw#QR-@CthDYePaoMFmGQp4in=(lt+20Wc7iIyX`}`P~V}6Wloc;_%~BNE}E!h zaeMR%M2p-nHU@_xdB~j-yAAvj@R$lM*FK*>Hx#wmhcR^d?H`@z)UIFR^6v9?fRYYK z9nW%W+q090s3A%zJ$CPB7k?;@vtTB(HtT-@4IT}FdGb+h;igYiB_GUcK`<=!h+lUuY7DYVkMI|1T58ItyQ+_U*W4X0q9(lN`0!~|+~~WviA#&bJWa%-O(2=hF#y6ZpW5`m+0oy$xfr4k>aRlNlf@&| zCK2LD2NA3|>bk+Mh)mV<(DrB}Q^yQ89PS?9 zNfN!8A}c~;y~-QwTN&e^1$9Xt>3gVD3QG*97dONj-_rMUU9doD85 z|9N?}Su%lt?>e^UP=?aB2}+N?RCz^FT-Qf@)`LWeSRsBSOkf1~6%aT;tXm%H-vezUyYAiT^rJBlc(?e8=>l02QetX1)VN;$Ta^s%vo>PX)mvFf?;!m1w72o0q)=G|rkS6W z_&v4+$;-yBx$=j+Z~RF#*6}V}3YEV#Z2c7j$z_~iAM{0Jn=^4rkO;*Jd?ikU5f`SR z_sK#_xWH6W;(%t5>R!mtQrFyGGMaqYFVpmAQdZ16UW-Ra%Pu$P$nZVxZXfmtracL0 z_Dx0jsI~$hjF`vnGRi(%#sY`o4YL`6Gv1aW6rQAbwzuT+tg-{&STQI-f^V&D{?vxS zX^J0z!xP^rI3bi+4(|R`NR_x-%etY%(2uIb{|84@QY7O31{^Zha!vwpNEPlL&PaA~@kxujk+qlgaj!+3M z-^?IE8KfNChWCb+s&ae@&%$-2Bm3YSJkfnl<);lKPbL^fwp0)d7rCY6&3=94yn=lW zgoXvRp55A$=TZI@x0Tgsv@u{B#g>==TckgUUZ0Ea#=HIl(^kLm>&ZC+URtU=&7>b% zL-5WI{wZW&(DT;%=9jbKfk*+Y0&IbfG&AnrbhneU|!+wHDklMgx43}Y{HAoygqTf z=71fAktl;XBnvav0rzBaqJu7bWIg2aux8P+HiVhHJyF22O@~x``-q(9uI60l7V#tZR;X;w!4WGc#G)Bkkxd^vpxhDdlxMae3w2DCz zKvWHc&q)i-p#e-`v8FC4j@q~Zs10=6G2lTJAxgPGVZ=BUG6|#^5-$iezI|Zri~2kV zuwWEL_Yh4eIJ8!f7C;NqqsC~$l0H%K{T^=q&Vkc7G(-l{g9fAC(BHE8`u5yeOlD!X zg1ZbISj7*J1fvhSH#MLJawg}KBAo?cjYq!UZ%#8!d4DL_!>eH`8IMo}8%4+v&afm{ zSoXC-(Go9g9awW)&d0>G4+~Ys-v)F+2DpY%E*LT0>%k<8ts=NxVU$TJ3~3yN9Dc8E-eA=~5LlyjP*U6~pIe{zimCUM z>r#ogzOK$ocw6wE+FK2bi8FfMLdMt&o2{`dzxLJaM8kn7!0%^6+yU_!+&d>8btNzR zX)|SD?G<5CYk*}_%_B8$---RYB%nt?stRtk*tO;pkO}mQ?AxQp`e{1ef6eJ;^V>>u z!t=%BEdG7d=}(~WSC3`pMBfNPtpHU{&yp$4ZQg3uRO7e|!B!cX5?rxCdOk^(#IX@v zO`hwyj`!6w;%PRtawd+IC|yiBU-TYjn^YV4Op0f)PK+XMpp)s)oM2##3vwTNF81_~ z4jki;oywhtLT$=aJV6TTqUMV__iL@xO)+BF?1QWC(=ZkqaF=&m5E4Ik2&kxNl Date: Thu, 17 Apr 2025 08:18:52 +0200 Subject: [PATCH 32/32] Updates for Readme --- helm/README.md | 17 +++++++++++++++-- helm/pgwatch/README.md | 17 +++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/helm/README.md b/helm/README.md index 36413d2..5c5c186 100644 --- a/helm/README.md +++ b/helm/README.md @@ -15,7 +15,17 @@ To use the Helm-Charts, you can either patch the repo onto your local system or In either case, please familiarise yourself with the relevant values files before use and create a custom variant to set up pgWatch according to your preferences in your environment. ### Helm-Repository -comming soon +```sh +# Add Helm-Repo +helm repo add pgwatch https://cybertec-postgresql.github.io/pgwatch-charts +helm repo update + +# Install helm-Chart +helm install pgwatch pgwatch/pgwatch --values custom-values.yaml + +# Upgrade helm-Chart +helm upgrade pgwatch pgwatch/pgwatch --values custom-values.yaml +``` ### git clone ```sh @@ -23,9 +33,12 @@ git clone https://github.com/cybertec-postgresql/pgwatch-charts.git cd pgwatch-chart/helm/pgwatch -// Install helm-Chart +# Install helm-Chart helm install pgwatch -n pgwatch -f custom-values.yaml . +# Upgrade Helm-Chart +helm upgrade pgwatch -n pgwatch -f custom-values.yaml . + ``` ### Custom Values-file diff --git a/helm/pgwatch/README.md b/helm/pgwatch/README.md index 3195f3a..d982378 100644 --- a/helm/pgwatch/README.md +++ b/helm/pgwatch/README.md @@ -13,7 +13,17 @@ To use the Helm-Charts, you can either patch the repo onto your local system or In either case, please familiarise yourself with the relevant values files before use and create a custom variant to set up pgWatch according to your preferences in your environment. ### Helm-Repository -comming soon +```sh +# Add Helm-Repo +helm repo add pgwatch https://cybertec-postgresql.github.io/pgwatch-charts +helm repo update + +# Install helm-Chart +helm install pgwatch pgwatch/pgwatch --values custom-values.yaml + +# Upgrade helm-Chart +helm upgrade pgwatch pgwatch/pgwatch --values custom-values.yaml +``` ### git clone ```sh @@ -21,9 +31,12 @@ git clone https://github.com/cybertec-postgresql/pgwatch-charts.git cd pgwatch-chart/helm/pgwatch -// Install helm-Chart +# Install helm-Chart helm install pgwatch -n pgwatch -f custom-values.yaml . +# Upgrade Helm-Chart +helm upgrade pgwatch -n pgwatch -f custom-values.yaml . + ``` ## customisation