[system/{diskio,network}]: Add alerting rule templates (#15998)

Author: shmsr

packages/system/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.8.0"
+  changes:
+    - description: Add alerting rule templates (for diskio and network datastreams)
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15998
 - version: "2.7.2"
   changes:
     - description: Fixed parsing of SidList field in Windows Security event 4908 (Special Groups Logon table modified) by normalizing whitespace separators.

packages/system/kibana/alerting_rule_template/system-disk-io-saturation.json

@@ -0,0 +1,30 @@
+{
+    "id": "system-disk-io-saturation",
+    "type": "alerting_rule_template",
+    "attributes": {
+        "name": "[System] Disk I/O Saturation",
+        "tags": [
+            "System"
+        ],
+        "ruleTypeId": ".es-query",
+        "schedule": {
+            "interval": "1m"
+        },
+        "params": {
+            "searchType": "esqlQuery",
+            "timeWindowSize": 5,
+            "timeWindowUnit": "m",
+            "esqlQuery": {
+                "esql": "// Alert when disk devices show sustained saturation via utilization and queue depth metrics.\n// Groups by host/device, enforces minimum work, and uses Little's Law to estimate queue depth.\n// Adjust utilization or queue depth thresholds in the final WHERE clause as needed.\nFROM metrics-system.diskio-*\n| WHERE `system.diskio.name` NOT RLIKE \"(loop|ram|sr|dm-)\"\n| STATS\nmax_io_time = MAX(`system.diskio.io.time`::LONG),\nmin_io_time = MIN(`system.diskio.io.time`::LONG),\nmax_read_time = MAX(`system.diskio.read.time`::LONG),\nmin_read_time = MIN(`system.diskio.read.time`::LONG),\nmax_write_time = MAX(`system.diskio.write.time`::LONG),\nmin_write_time = MIN(`system.diskio.write.time`::LONG),\nmax_read_ops = MAX(`system.diskio.read.count`::LONG),\nmin_read_ops = MIN(`system.diskio.read.count`::LONG),\nmax_write_ops = MAX(`system.diskio.write.count`::LONG),\nmin_write_ops = MIN(`system.diskio.write.count`::LONG),\nfirst_ts = MIN(@timestamp),\nlast_ts = MAX(@timestamp)\nBY `host.name`, `system.diskio.name`\n| EVAL elapsed_ms = TO_DOUBLE(TO_LONG(last_ts) - TO_LONG(first_ts))\n| WHERE elapsed_ms >= 60000\n| EVAL io_time_delta = CASE(max_io_time >= min_io_time, max_io_time - min_io_time, 0)\n| EVAL read_time_delta = CASE(max_read_time >= min_read_time, max_read_time - min_read_time, 0)\n| EVAL write_time_delta = CASE(max_write_time >= min_write_time, max_write_time - min_write_time, 0)\n| EVAL read_ops_delta = CASE(max_read_ops >= min_read_ops, max_read_ops - min_read_ops, 0)\n| EVAL write_ops_delta = CASE(max_write_ops >= min_write_ops, max_write_ops - min_write_ops, 0)\n| EVAL total_ops = read_ops_delta + write_ops_delta\n| WHERE total_ops >= 100\n| EVAL io_util_pct = CASE(elapsed_ms > 0, 100.0 * io_time_delta / elapsed_ms, 0.0)\n| EVAL avg_iops = CASE(elapsed_ms > 0, total_ops / (elapsed_ms / 1000.0), 0.0)\n| EVAL total_io_time_ms = read_time_delta + write_time_delta\n| EVAL avg_latency_ms = CASE(\ntotal_ops > 0 AND total_io_time_ms > 0,\ntotal_io_time_ms / TO_DOUBLE(total_ops),\n0\n)\n| EVAL est_queue_depth = CASE(\navg_latency_ms > 0 AND avg_iops > 0,\n(avg_iops * avg_latency_ms) / 1000,\n0\n)\n| WHERE io_util_pct > 90\nOR est_queue_depth > 10\nOR (io_util_pct > 70 AND est_queue_depth > 5)"
+            },
+            "groupBy": "row",
+            "timeField": "@timestamp"
+        },
+        "alertDelay": {
+            "active": 3
+        }
+    },
+    "managed": true,
+    "coreMigrationVersion": "8.8.0",
+    "typeMigrationVersion": "10.1.0"
+}

packages/system/kibana/alerting_rule_template/system-high-disk-io-latency.json

@@ -0,0 +1,30 @@
+{
+    "id": "system-high-disk-io-latency",
+    "type": "alerting_rule_template",
+    "attributes": {
+        "name": "[System] High Disk I/O Latency",
+        "tags": [
+            "System"
+        ],
+        "ruleTypeId": ".es-query",
+        "schedule": {
+            "interval": "1m"
+        },
+        "params": {
+            "searchType": "esqlQuery",
+            "timeWindowSize": 5,
+            "timeWindowUnit": "m",
+            "esqlQuery": {
+                "esql": "// Alert when average disk I/O latency exceeds storage-type thresholds within the configured lookback window.\n// Groups by host and device, excludes loop/ram/sr/dm devices, and clamps counter resets.\n// Adjust latency thresholds in the final WHERE clause as needed.\nFROM metrics-system.diskio-*\n| WHERE `system.diskio.name` NOT RLIKE \"(loop|ram|sr|dm-)\"\n| STATS\nmax_read_time = MAX(`system.diskio.read.time`::LONG),\nmin_read_time = MIN(`system.diskio.read.time`::LONG),\nmax_write_time = MAX(`system.diskio.write.time`::LONG),\nmin_write_time = MIN(`system.diskio.write.time`::LONG),\nmax_read_ops = MAX(`system.diskio.read.count`::LONG),\nmin_read_ops = MIN(`system.diskio.read.count`::LONG),\nmax_write_ops = MAX(`system.diskio.write.count`::LONG),\nmin_write_ops = MIN(`system.diskio.write.count`::LONG),\nfirst_ts = MIN(@timestamp),\nlast_ts = MAX(@timestamp)\nBY `host.name`, `system.diskio.name`\n| EVAL elapsed_ms = TO_DOUBLE(TO_LONG(last_ts) - TO_LONG(first_ts))\n| WHERE elapsed_ms >= 60000\n| EVAL read_time_delta = CASE(max_read_time >= min_read_time, max_read_time - min_read_time, 0)\n| EVAL write_time_delta = CASE(max_write_time >= min_write_time, max_write_time - min_write_time, 0)\n| EVAL read_ops_delta = CASE(max_read_ops >= min_read_ops, max_read_ops - min_read_ops, 0)\n| EVAL write_ops_delta = CASE(max_write_ops >= min_write_ops, max_write_ops - min_write_ops, 0)\n| EVAL total_ops = read_ops_delta + write_ops_delta\n| WHERE total_ops >= 100\n| EVAL io_time_ms = read_time_delta + write_time_delta\n| WHERE io_time_ms > 0\n| EVAL avg_latency_ms = io_time_ms / TO_DOUBLE(total_ops)\n| EVAL avg_iops = CASE(elapsed_ms > 0, total_ops / (elapsed_ms / 1000.0), 0.0)\n| EVAL device_type = CASE(\n`system.diskio.name` RLIKE \"^nvme[0-9]\", \"ssd\",\navg_latency_ms < 3 AND avg_iops > 100, \"ssd\",\navg_latency_ms > 15, \"hdd\",\n\"unknown\"\n)\n| WHERE (device_type == \"ssd\" AND avg_latency_ms > 5)\nOR (device_type == \"hdd\" AND avg_latency_ms > 30)\nOR (device_type == \"unknown\" AND avg_latency_ms > 10)"
+            },
+            "groupBy": "row",
+            "timeField": "@timestamp"
+        },
+        "alertDelay": {
+            "active": 3
+        }
+    },
+    "managed": true,
+    "coreMigrationVersion": "8.8.0",
+    "typeMigrationVersion": "10.1.0"
+}

packages/system/kibana/alerting_rule_template/system-high-network-error-rate.json

@@ -0,0 +1,30 @@
+{
+    "id": "system-high-network-error-rate",
+    "type": "alerting_rule_template",
+    "attributes": {
+        "name": "[System] High Network Error Rate",
+        "tags": [
+            "System"
+        ],
+        "ruleTypeId": ".es-query",
+        "schedule": {
+            "interval": "1m"
+        },
+        "params": {
+            "searchType": "esqlQuery",
+            "timeWindowSize": 5,
+            "timeWindowUnit": "m",
+            "esqlQuery": {
+                "esql": "// Alert when network interfaces show sustained error rates (>5 errors/sec or >0.5% of packets).\n// Excludes loopback/container interfaces, enforces minimum traffic, and clamps counter resets.\n// Tune thresholds in the final WHERE clause for your environment.\nFROM metrics-system.network-*\n| WHERE `system.network.name` NOT RLIKE \"^(lo|docker|veth|br-)\"\n| STATS\n  max_in_errors  = MAX(`system.network.in.errors`::LONG),\n  min_in_errors  = MIN(`system.network.in.errors`::LONG),\n  max_out_errors = MAX(`system.network.out.errors`::LONG),\n  min_out_errors = MIN(`system.network.out.errors`::LONG),\n  max_in_packets = MAX(`system.network.in.packets`::LONG),\n  min_in_packets = MIN(`system.network.in.packets`::LONG),\n  max_out_packets = MAX(`system.network.out.packets`::LONG),\n  min_out_packets = MIN(`system.network.out.packets`::LONG),\n  first_ts = MIN(@timestamp),\n  last_ts  = MAX(@timestamp)\n  BY `host.name`, `system.network.name`\n| EVAL elapsed_ms  = TO_DOUBLE(TO_LONG(last_ts) - TO_LONG(first_ts))\n| EVAL elapsed_sec = CASE(elapsed_ms > 0, elapsed_ms / 1000.0, 0.0)\n| WHERE elapsed_sec >= 60\n| EVAL in_error_delta   = CASE(max_in_errors  >= min_in_errors,  max_in_errors  - min_in_errors,  0)\n| EVAL out_error_delta  = CASE(max_out_errors >= min_out_errors, max_out_errors - min_out_errors, 0)\n| EVAL in_packet_delta  = CASE(max_in_packets >= min_in_packets, max_in_packets - min_in_packets, 0)\n| EVAL out_packet_delta = CASE(max_out_packets >= min_out_packets, max_out_packets - min_out_packets, 0)\n| EVAL total_packets = in_packet_delta + out_packet_delta\n| WHERE total_packets >= 1000\n| EVAL in_error_rate  = CASE(elapsed_sec > 0, in_error_delta  / elapsed_sec, 0.0)\n| EVAL out_error_rate = CASE(elapsed_sec > 0, out_error_delta / elapsed_sec, 0.0)\n| EVAL in_error_pct  = CASE(in_packet_delta  > 100, 100.0 * in_error_delta  / TO_DOUBLE(in_packet_delta),  0.0)\n| EVAL out_error_pct = CASE(out_packet_delta > 100, 100.0 * out_error_delta / TO_DOUBLE(out_packet_delta), 0.0)\n| WHERE in_error_rate > 5\n  OR out_error_rate > 5\n  OR in_error_pct  > 0.5\n  OR out_error_pct > 0.5"
+            },
+            "groupBy": "row",
+            "timeField": "@timestamp"
+        },
+        "alertDelay": {
+            "active": 3
+        }
+    },
+    "managed": true,
+    "coreMigrationVersion": "8.8.0",
+    "typeMigrationVersion": "10.1.0"
+}

packages/system/kibana/alerting_rule_template/system-high-packet-drop-rate.json

@@ -0,0 +1,30 @@
+{
+    "id": "system-high-packet-drop-rate",
+    "type": "alerting_rule_template",
+    "attributes": {
+        "name": "[System] High Packet Drop Rate",
+        "tags": [
+            "System"
+        ],
+        "ruleTypeId": ".es-query",
+        "schedule": {
+            "interval": "1m"
+        },
+        "params": {
+            "searchType": "esqlQuery",
+            "timeWindowSize": 5,
+            "timeWindowUnit": "m",
+            "esqlQuery": {
+                "esql": "// Alert when interfaces drop >25 packets/sec or more than 1% of traffic over the configured time window.\n// Excludes loopback/container interfaces, requires minimum packet volume, and guards against counter resets.\n// Adjust absolute or percentage thresholds in the final WHERE clause as needed.\nFROM metrics-system.network-*\n| WHERE `system.network.name` NOT RLIKE \"^(lo|docker|veth|br-)\"\n| STATS\nmax_in_dropped = MAX(`system.network.in.dropped`::LONG),\nmin_in_dropped = MIN(`system.network.in.dropped`::LONG),\nmax_out_dropped = MAX(`system.network.out.dropped`::LONG),\nmin_out_dropped = MIN(`system.network.out.dropped`::LONG),\nmax_in_packets = MAX(`system.network.in.packets`::LONG),\nmin_in_packets = MIN(`system.network.in.packets`::LONG),\nmax_out_packets = MAX(`system.network.out.packets`::LONG),\nmin_out_packets = MIN(`system.network.out.packets`::LONG),\nfirst_ts = MIN(@timestamp),\nlast_ts = MAX(@timestamp)\nBY `host.name`, `system.network.name`\n| EVAL elapsed_ms = TO_DOUBLE(TO_LONG(last_ts) - TO_LONG(first_ts))\n| EVAL elapsed_sec = CASE(elapsed_ms > 0, elapsed_ms / 1000.0, 0.0)\n| WHERE elapsed_sec >= 60\n| EVAL in_drop_delta = CASE(max_in_dropped >= min_in_dropped, max_in_dropped - min_in_dropped, 0)\n| EVAL out_drop_delta = CASE(max_out_dropped >= min_out_dropped, max_out_dropped - min_out_dropped, 0)\n| EVAL in_packet_delta = CASE(max_in_packets >= min_in_packets, max_in_packets - min_in_packets, 0)\n| EVAL out_packet_delta = CASE(max_out_packets >= min_out_packets, max_out_packets - min_out_packets, 0)\n| EVAL total_packets = in_packet_delta + out_packet_delta\n| WHERE total_packets >= 1000\n| EVAL in_drop_rate = CASE(elapsed_sec > 0, in_drop_delta / elapsed_sec, 0.0)\n| EVAL out_drop_rate = CASE(elapsed_sec > 0, out_drop_delta / elapsed_sec, 0.0)\n| EVAL in_drop_pct = CASE(in_packet_delta > 100, 100.0 * in_drop_delta / TO_DOUBLE(in_packet_delta), 0.0)\n| EVAL out_drop_pct = CASE(out_packet_delta > 100, 100.0 * out_drop_delta / TO_DOUBLE(out_packet_delta), 0.0)\n| WHERE in_drop_rate > 25\nOR out_drop_rate > 25\nOR in_drop_pct > 1\nOR out_drop_pct > 1"
+            },
+            "groupBy": "row",
+            "timeField": "@timestamp"
+        },
+        "alertDelay": {
+            "active": 3
+        }
+    },
+    "managed": true,
+    "coreMigrationVersion": "8.8.0",
+    "typeMigrationVersion": "10.1.0"
+}

packages/system/manifest.yml

@@ -1,7 +1,7 @@
-format_version: 3.0.2
+format_version: 3.4.0
 name: system
 title: System
-version: "2.7.2"
+version: "2.8.0"
 description: Collect system logs and metrics from your servers with Elastic Agent.
 type: integration
 categories:
@@ -12,7 +12,7 @@ categories:
   - observability
 conditions:
   kibana:
-    version: "^9.2.0"
+    version: "^9.2.1"
 screenshots:
   - src: /img/policy-and-object-monitoring-dashboard.png
     title: policy and object monitoring

packages/system/validation.yml

@@ -2,3 +2,4 @@ errors:
   exclude_checks:
     - SVR00002 # expected filter in dashboard.
     - SVR00004 # references found in dashboard.
+    - JSE00001