From 9e96fa2b3b6f2beb2cfe16edac898ae4f08fb68e Mon Sep 17 00:00:00 2001 From: "avi@robusta.dev" Date: Sun, 19 Jan 2025 14:35:28 +0200 Subject: [PATCH 1/4] auto investigate playbook - actions: - auto_ask_holmes: {} triggers: - on_prometheus_alert: alert_name: PodNotReady - on_pod_crash_loop: restart_reason: "CrashLoopBackOff" --- .../core/playbooks/internal/ai_integration.py | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/robusta/core/playbooks/internal/ai_integration.py b/src/robusta/core/playbooks/internal/ai_integration.py index 4c580b266..0d4550cb3 100644 --- a/src/robusta/core/playbooks/internal/ai_integration.py +++ b/src/robusta/core/playbooks/internal/ai_integration.py @@ -29,6 +29,7 @@ ) from robusta.core.schedule.model import FixedDelayRepeat from robusta.integrations.kubernetes.autogenerated.events import KubernetesAnyChangeEvent +from robusta.integrations.prometheus.models import PrometheusKubernetesAlert from robusta.integrations.prometheus.utils import HolmesDiscovery from robusta.utils.error_codes import ActionException, ErrorCodes @@ -40,6 +41,32 @@ def build_investigation_title(params: AIInvestigateParams) -> str: return params.context.get("issue_type", "unknown health issue") +@action +def auto_ask_holmes(event: ExecutionBaseEvent): + subject: FindingSubject = event.get_subject() + resource = ResourceInfo( + name=subject.name, + namespace=subject.namespace, + kind=str(subject.subject_type.value), + node=subject.node, + container=subject.container, + ) + issue_type = event.alert_name if isinstance(event, PrometheusKubernetesAlert) else type(event).__name__ + logging.warning(issue_type) + source = event.get_source() + + context = { + "issue_type": issue_type, + "source": str(source.value), + "labels": subject.labels, + } + + action_params = AIInvestigateParams( + resource=resource, investigation_type="issue", ask="Why is this alert firing?", context=context + ) + ask_holmes(event, params=action_params) + + @action def ask_holmes(event: ExecutionBaseEvent, params: AIInvestigateParams): holmes_url = HolmesDiscovery.find_holmes_url(params.holmes_url) @@ -167,7 +194,9 @@ def build_conversation_title(params: HolmesConversationParams) -> str: def add_labels_to_ask(params: HolmesConversationParams) -> str: - label_string = f"the alert has the following labels: {params.context.get('labels')}" if params.context.get("labels") else "" + label_string = ( + f"the alert has the following labels: {params.context.get('labels')}" if params.context.get("labels") else "" + ) ask = f"{params.ask}, {label_string}" if label_string else params.ask logging.debug(f"holmes ask query: {ask}") return ask From b1404e487212b1fdc575d2a165057e53ad244c73 Mon Sep 17 00:00:00 2001 From: "avi@robusta.dev" Date: Mon, 20 Jan 2025 09:59:35 +0200 Subject: [PATCH 2/4] comment added --- src/robusta/core/playbooks/internal/ai_integration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/robusta/core/playbooks/internal/ai_integration.py b/src/robusta/core/playbooks/internal/ai_integration.py index 0d4550cb3..b8fe178f0 100644 --- a/src/robusta/core/playbooks/internal/ai_integration.py +++ b/src/robusta/core/playbooks/internal/ai_integration.py @@ -43,6 +43,9 @@ def build_investigation_title(params: AIInvestigateParams) -> str: @action def auto_ask_holmes(event: ExecutionBaseEvent): + """ + Runs holmes investigation on an alert/event. + """ subject: FindingSubject = event.get_subject() resource = ResourceInfo( name=subject.name, From 03a3e11fde9339f680d1809b4c105c4cde5b06ff Mon Sep 17 00:00:00 2001 From: "avi@robusta.dev" Date: Thu, 23 Jan 2025 10:11:48 +0200 Subject: [PATCH 3/4] fixed auto investigate finding --- src/robusta/core/playbooks/internal/ai_integration.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/robusta/core/playbooks/internal/ai_integration.py b/src/robusta/core/playbooks/internal/ai_integration.py index b8fe178f0..96f2c7c5b 100644 --- a/src/robusta/core/playbooks/internal/ai_integration.py +++ b/src/robusta/core/playbooks/internal/ai_integration.py @@ -1,5 +1,6 @@ import json import logging +from typing import Optional import requests @@ -15,6 +16,7 @@ from robusta.core.playbooks.actions_registry import action from robusta.core.reporting import Finding, FindingSubject from robusta.core.reporting.base import EnrichmentType +from robusta.core.reporting.blocks import MarkdownBlock from robusta.core.reporting.consts import FindingSubjectType, FindingType from robusta.core.reporting.holmes import ( HolmesChatRequest, @@ -67,11 +69,11 @@ def auto_ask_holmes(event: ExecutionBaseEvent): action_params = AIInvestigateParams( resource=resource, investigation_type="issue", ask="Why is this alert firing?", context=context ) - ask_holmes(event, params=action_params) + ask_holmes(event, params=action_params, create_new_finding=False) @action -def ask_holmes(event: ExecutionBaseEvent, params: AIInvestigateParams): +def ask_holmes(event: ExecutionBaseEvent, params: AIInvestigateParams, create_new_finding: bool = True): holmes_url = HolmesDiscovery.find_holmes_url(params.holmes_url) if not holmes_url: raise ActionException(ErrorCodes.HOLMES_DISCOVERY_FAILED, "Robusta couldn't connect to the Holmes client.") @@ -93,6 +95,10 @@ def ask_holmes(event: ExecutionBaseEvent, params: AIInvestigateParams): result.raise_for_status() holmes_result = HolmesResult(**json.loads(result.text)) + + if not create_new_finding: + event.add_enrichment([MarkdownBlock(f"{holmes_result.analysis}")]) + title_suffix = ( f" on {params.resource.name}" if params.resource and params.resource.name and params.resource.name.lower() != "unresolved" From 155dd4e7877b6279956b46a3ccecffff41224c84 Mon Sep 17 00:00:00 2001 From: Arik Alon Date: Sat, 25 Jan 2025 02:09:05 +0200 Subject: [PATCH 4/4] Support sending text files and tables to the webhook sink --- src/robusta/core/sinks/webhook/webhook_sink.py | 7 +++++++ src/robusta/core/sinks/webhook/webhook_sink_params.py | 2 ++ 2 files changed, 9 insertions(+) diff --git a/src/robusta/core/sinks/webhook/webhook_sink.py b/src/robusta/core/sinks/webhook/webhook_sink.py index c56127492..cfaf25c08 100644 --- a/src/robusta/core/sinks/webhook/webhook_sink.py +++ b/src/robusta/core/sinks/webhook/webhook_sink.py @@ -5,6 +5,7 @@ import requests +from robusta.core.reporting import TableBlock, FileBlock from robusta.core.reporting import HeaderBlock, JsonBlock, KubernetesDiffBlock, ListBlock, MarkdownBlock from robusta.core.reporting.base import BaseBlock, Finding from robusta.core.sinks.sink_base import SinkBase @@ -25,6 +26,8 @@ def __init__(self, sink_config: WebhookSinkConfigWrapper, registry): ) self.size_limit = sink_config.webhook_sink.size_limit self.slack_webhook = sink_config.webhook_sink.slack_webhook + self.send_table_block = sink_config.webhook_sink.table_blocks + self.send_file_block = sink_config.webhook_sink.file_blocks def write_finding(self, finding: Finding, platform_enabled: bool): if self.format == "text": @@ -134,6 +137,10 @@ def __to_unformatted_text(cls, block: BaseBlock) -> List[str]: lines.append(cls.__to_clear_text(block.text)) elif isinstance(block, JsonBlock): lines.append(block.json_str) + elif isinstance(block, TableBlock) and cls.send_table_block: + lines.append(block.to_table_string()) + elif isinstance(block, FileBlock) and cls.send_file_block and block.is_text_file(): + lines.append(str(block.contents)) elif isinstance(block, KubernetesDiffBlock): for diff in block.diffs: lines.append(f"*{'.'.join(diff.path)}*: {diff.other_value} ==> {diff.value}") diff --git a/src/robusta/core/sinks/webhook/webhook_sink_params.py b/src/robusta/core/sinks/webhook/webhook_sink_params.py index 9462919ae..e12ef59ae 100644 --- a/src/robusta/core/sinks/webhook/webhook_sink_params.py +++ b/src/robusta/core/sinks/webhook/webhook_sink_params.py @@ -10,6 +10,8 @@ class WebhookSinkParams(SinkBaseParams): authorization: SecretStr = None format: str = "text" slack_webhook: bool = False + table_blocks: bool = False + file_blocks: bool = False @classmethod def _get_sink_type(cls):