diff --git a/src/robusta/core/playbooks/internal/ai_integration.py b/src/robusta/core/playbooks/internal/ai_integration.py index 80c17bafd..10efe4ba1 100644 --- a/src/robusta/core/playbooks/internal/ai_integration.py +++ b/src/robusta/core/playbooks/internal/ai_integration.py @@ -1,3 +1,4 @@ +import base64 import json import logging import re @@ -17,7 +18,9 @@ ) from robusta.core.model.events import ExecutionBaseEvent from robusta.core.playbooks.actions_registry import action -from robusta.core.playbooks.prometheus_enrichment_utils import build_chart_from_prometheus_result +from robusta.core.playbooks.prometheus_enrichment_utils import ( + build_chart_from_prometheus_result, +) from robusta.core.reporting import Finding, FindingSubject from robusta.core.reporting.base import EnrichmentType from robusta.core.reporting.consts import FindingSubjectType, FindingType @@ -35,11 +38,21 @@ HolmesWorkloadHealthRequest, ) from robusta.core.reporting.utils import convert_svg_to_png +from robusta.core.stream.utils import ( + create_sse_message, + parse_sse_event_type, + parse_sse_data, + StreamEvents, +) from robusta.core.schedule.model import FixedDelayRepeat -from robusta.integrations.kubernetes.autogenerated.events import KubernetesAnyChangeEvent +from robusta.integrations.kubernetes.autogenerated.events import ( + KubernetesAnyChangeEvent, +) from robusta.integrations.prometheus.utils import HolmesDiscovery from robusta.utils.error_codes import ActionException, ErrorCodes +GRAPH_TOOLS = ["execute_prometheus_range_query", "query_datadog_metrics"] + def build_investigation_title(params: AIInvestigateParams) -> str: if params.investigation_type == "analyze_problems": @@ -50,22 +63,36 @@ def build_investigation_title(params: AIInvestigateParams) -> str: def handle_holmes_error(e: Exception) -> NoReturn: if isinstance(e, requests.ConnectionError): - raise ActionException(ErrorCodes.HOLMES_CONNECTION_ERROR, "Holmes endpoint is currently unreachable.") + raise ActionException( + ErrorCodes.HOLMES_CONNECTION_ERROR, + "Holmes endpoint is currently unreachable.", + ) elif isinstance(e, requests.HTTPError): if e.response.status_code == 401 and "invalid_api_key" in e.response.text: - raise ActionException(ErrorCodes.HOLMES_REQUEST_ERROR, "Holmes invalid api key.") + raise ActionException( + ErrorCodes.HOLMES_REQUEST_ERROR, "Holmes invalid api key." + ) elif e.response.status_code == 429: - raise ActionException(ErrorCodes.HOLMES_RATE_LIMIT_EXCEEDED, "Holmes rate limit exceeded.") - raise ActionException(ErrorCodes.HOLMES_REQUEST_ERROR, "Holmes internal configuration error.") + raise ActionException( + ErrorCodes.HOLMES_RATE_LIMIT_EXCEEDED, "Holmes rate limit exceeded." + ) + raise ActionException( + ErrorCodes.HOLMES_REQUEST_ERROR, "Holmes internal configuration error." + ) else: - raise ActionException(ErrorCodes.HOLMES_UNEXPECTED_ERROR, "An unexpected error occured.") + raise ActionException( + ErrorCodes.HOLMES_UNEXPECTED_ERROR, "An unexpected error occured." + ) @action def ask_holmes(event: ExecutionBaseEvent, params: AIInvestigateParams): holmes_url = HolmesDiscovery.find_holmes_url(params.holmes_url) if not holmes_url: - raise ActionException(ErrorCodes.HOLMES_DISCOVERY_FAILED, "Robusta couldn't connect to the Holmes client.") + raise ActionException( + ErrorCodes.HOLMES_DISCOVERY_FAILED, + "Robusta couldn't connect to the Holmes client.", + ) investigation__title = build_investigation_title(params) subject = params.resource.dict() if params.resource else {} @@ -73,7 +100,9 @@ def ask_holmes(event: ExecutionBaseEvent, params: AIInvestigateParams): try: params.ask = add_labels_to_ask(params) holmes_req = HolmesRequest( - source=params.context.get("source", "unknown source") if params.context else "unknown source", + source=params.context.get("source", "unknown source") + if params.context + else "unknown source", title=investigation__title, subject=subject, context=params.context if params.context else {}, @@ -99,13 +128,17 @@ def ask_holmes(event: ExecutionBaseEvent, params: AIInvestigateParams): return else: - result = requests.post(f"{holmes_url}/api/investigate", data=holmes_req.json()) + result = requests.post( + f"{holmes_url}/api/investigate", data=holmes_req.json() + ) result.raise_for_status() holmes_result = HolmesResult(**json.loads(result.text)) title_suffix = ( f" on {params.resource.name}" - if params.resource and params.resource.name and params.resource.name.lower() != "unresolved" + if params.resource + and params.resource.name + and params.resource.name.lower() != "unresolved" else "" ) @@ -116,7 +149,9 @@ def ask_holmes(event: ExecutionBaseEvent, params: AIInvestigateParams): subject=FindingSubject( name=params.resource.name if params.resource else "", namespace=params.resource.namespace if params.resource else "", - subject_type=FindingSubjectType.from_kind(kind) if kind else FindingSubjectType.TYPE_NONE, + subject_type=FindingSubjectType.from_kind(kind) + if kind + else FindingSubjectType.TYPE_NONE, node=params.resource.node if params.resource else "", container=params.resource.container if params.resource else "", ), @@ -124,28 +159,37 @@ def ask_holmes(event: ExecutionBaseEvent, params: AIInvestigateParams): failure=False, ) finding.add_enrichment( - [HolmesResultsBlock(holmes_result=holmes_result)], enrichment_type=EnrichmentType.ai_analysis + [HolmesResultsBlock(holmes_result=holmes_result)], + enrichment_type=EnrichmentType.ai_analysis, ) event.add_finding(finding) except Exception as e: logging.exception( - f"Failed to get holmes analysis for {investigation__title} {params.context} {subject}", exc_info=True + f"Failed to get holmes analysis for {investigation__title} {params.context} {subject}", + exc_info=True, ) handle_holmes_error(e) @action -def holmes_workload_health(event: ExecutionBaseEvent, params: HolmesWorkloadHealthParams): +def holmes_workload_health( + event: ExecutionBaseEvent, params: HolmesWorkloadHealthParams +): holmes_url = HolmesDiscovery.find_holmes_url(params.holmes_url) if not holmes_url: - raise ActionException(ErrorCodes.HOLMES_DISCOVERY_FAILED, "Robusta couldn't connect to the Holmes client.") + raise ActionException( + ErrorCodes.HOLMES_DISCOVERY_FAILED, + "Robusta couldn't connect to the Holmes client.", + ) params.resource.cluster = event.get_context().cluster_name try: - result = requests.post(f"{holmes_url}/api/workload_health_check", data=params.json()) + result = requests.post( + f"{holmes_url}/api/workload_health_check", data=params.json() + ) result.raise_for_status() holmes_result = HolmesResult(**json.loads(result.text)) @@ -155,7 +199,9 @@ def holmes_workload_health(event: ExecutionBaseEvent, params: HolmesWorkloadHeal analysis = json.loads(holmes_result.analysis) healthy = analysis.get("workload_healthy") except Exception: - logging.exception("Error in holmes response format, analysis did not return the expected json format.") + logging.exception( + "Error in holmes response format, analysis did not return the expected json format." + ) pass if params.silent_healthy and healthy: @@ -179,22 +225,30 @@ def holmes_workload_health(event: ExecutionBaseEvent, params: HolmesWorkloadHeal failure=False, ) finding.add_enrichment( - [HolmesResultsBlock(holmes_result=holmes_result)], enrichment_type=EnrichmentType.ai_analysis + [HolmesResultsBlock(holmes_result=holmes_result)], + enrichment_type=EnrichmentType.ai_analysis, ) event.add_finding(finding) except Exception as e: - logging.exception(f"Failed to get holmes analysis for {params.resource}, {params.ask}", exc_info=True) + logging.exception( + f"Failed to get holmes analysis for {params.resource}, {params.ask}", + exc_info=True, + ) handle_holmes_error(e) def build_conversation_title(params: HolmesConversationParams) -> str: - return f"{params.resource}, {params.ask} for issue '{params.context.robusta_issue_id}'" + return ( + f"{params.resource}, {params.ask} for issue '{params.context.robusta_issue_id}'" + ) def add_labels_to_ask(params: HolmesConversationParams) -> str: label_string = ( - f"the alert has the following labels: {params.context.get('labels')}" if params.context.get("labels") else "" + f"the alert has the following labels: {params.context.get('labels')}" + if params.context.get("labels") + else "" ) ask = f"{params.ask}, {label_string}" if label_string else params.ask logging.debug(f"holmes ask query: {ask}") @@ -206,14 +260,19 @@ def add_labels_to_ask(params: HolmesConversationParams) -> str: def holmes_conversation(event: ExecutionBaseEvent, params: HolmesConversationParams): holmes_url = HolmesDiscovery.find_holmes_url(params.holmes_url) if not holmes_url: - raise ActionException(ErrorCodes.HOLMES_DISCOVERY_FAILED, "Robusta couldn't connect to the Holmes client.") + raise ActionException( + ErrorCodes.HOLMES_DISCOVERY_FAILED, + "Robusta couldn't connect to the Holmes client.", + ) conversation_title = build_conversation_title(params) try: holmes_req = HolmesConversationRequest( user_prompt=params.ask, - source=getattr(params.context, "source", "unknown source") if params.context else "unknown source", + source=getattr(params.context, "source", "unknown source") + if params.context + else "unknown source", resource=params.resource, conversation_type=params.conversation_type, context=params.context, @@ -243,13 +302,16 @@ def holmes_conversation(event: ExecutionBaseEvent, params: HolmesConversationPar failure=False, ) finding.add_enrichment( - [HolmesResultsBlock(holmes_result=holmes_result)], enrichment_type=EnrichmentType.ai_analysis + [HolmesResultsBlock(holmes_result=holmes_result)], + enrichment_type=EnrichmentType.ai_analysis, ) event.add_finding(finding) except Exception as e: - logging.exception(f"Failed to get holmes chat for {conversation_title}", exc_info=True) + logging.exception( + f"Failed to get holmes chat for {conversation_title}", exc_info=True + ) handle_holmes_error(e) @@ -258,7 +320,9 @@ class DelayedHealthCheckParams(HolmesWorkloadHealthParams): @action -def delayed_health_check(event: KubernetesAnyChangeEvent, action_params: DelayedHealthCheckParams): +def delayed_health_check( + event: KubernetesAnyChangeEvent, action_params: DelayedHealthCheckParams +): """ runs a holmes workload health action with a delay """ @@ -267,13 +331,19 @@ def delayed_health_check(event: KubernetesAnyChangeEvent, action_params: Delayed if not action_params.ask: action_params.ask = f"help me diagnose an issue with a workload {metadata.namespace}/{event.obj.kind}/{metadata.name} running in my Kubernetes cluster. Can you assist with identifying potential issues and pinpoint the root cause." - action_params.resource = ResourceInfo(name=metadata.name, namespace=metadata.namespace, kind=event.obj.kind) + action_params.resource = ResourceInfo( + name=metadata.name, namespace=metadata.namespace, kind=event.obj.kind + ) - logging.info(f"Scheduling health check. {metadata.name} delays: {action_params.delay_seconds}") + logging.info( + f"Scheduling health check. {metadata.name} delays: {action_params.delay_seconds}" + ) event.get_scheduler().schedule_action( action_func=holmes_workload_health, task_id=f"health_check_{metadata.name}_{metadata.namespace}", - scheduling_params=FixedDelayRepeat(repeat=1, seconds_delay=action_params.delay_seconds), + scheduling_params=FixedDelayRepeat( + repeat=1, seconds_delay=action_params.delay_seconds + ), named_sinks=event.named_sinks, action_params=action_params, replace_existing=True, @@ -285,7 +355,10 @@ def delayed_health_check(event: KubernetesAnyChangeEvent, action_params: Delayed def holmes_issue_chat(event: ExecutionBaseEvent, params: HolmesIssueChatParams): holmes_url = HolmesDiscovery.find_holmes_url(params.holmes_url) if not holmes_url: - raise ActionException(ErrorCodes.HOLMES_DISCOVERY_FAILED, "Robusta couldn't connect to the Holmes client.") + raise ActionException( + ErrorCodes.HOLMES_DISCOVERY_FAILED, + "Robusta couldn't connect to the Holmes client.", + ) conversation_title = build_conversation_title(params) params_resource_kind = params.resource.kind or "" @@ -320,13 +393,16 @@ def holmes_issue_chat(event: ExecutionBaseEvent, params: HolmesIssueChatParams): failure=False, ) finding.add_enrichment( - [HolmesChatResultsBlock(holmes_result=holmes_result)], enrichment_type=EnrichmentType.ai_analysis + [HolmesChatResultsBlock(holmes_result=holmes_result)], + enrichment_type=EnrichmentType.ai_analysis, ) event.add_finding(finding) except Exception as e: - logging.exception(f"Failed to get holmes chat for {conversation_title}", exc_info=True) + logging.exception( + f"Failed to get holmes chat for {conversation_title}", exc_info=True + ) handle_holmes_error(e) @@ -334,35 +410,42 @@ def holmes_issue_chat(event: ExecutionBaseEvent, params: HolmesIssueChatParams): def holmes_chat(event: ExecutionBaseEvent, params: HolmesChatParams): holmes_url = HolmesDiscovery.find_holmes_url(params.holmes_url) if not holmes_url: - raise ActionException(ErrorCodes.HOLMES_DISCOVERY_FAILED, "Robusta couldn't connect to the Holmes client.") + raise ActionException( + ErrorCodes.HOLMES_DISCOVERY_FAILED, + "Robusta couldn't connect to the Holmes client.", + ) cluster_name = event.get_context().cluster_name try: holmes_req = HolmesChatRequest( - ask=params.ask, - conversation_history=params.conversation_history, - model=params.model, - stream=params.stream, + ask=params.ask, + conversation_history=params.conversation_history, + model=params.model, + stream=params.stream, additional_system_prompt=params.additional_system_prompt, enable_tool_approval=params.enable_tool_approval, tool_decisions=params.tool_decisions, ) url = f"{holmes_url}/api/chat" if params.stream: - with requests.post( - url, - data=holmes_req.json(), - stream=True, - headers={"Connection": "keep-alive"}, - ) as resp: - resp.raise_for_status() - for line in resp.iter_content( - chunk_size=None, decode_unicode=True - ): # Avoid streaming chunks from holmes. send them as they arrive. - if line: - event.ws(data=line) - return + if params.render_graph_images: + stream_and_render_graphs(url, holmes_req, event) + return + else: + with requests.post( + url, + data=holmes_req.json(), + stream=True, + headers={"Connection": "keep-alive"}, + ) as resp: + resp.raise_for_status() + for line in resp.iter_content( + chunk_size=None, decode_unicode=True + ): # Avoid streaming chunks from holmes. send them as they arrive. + if line: + event.ws(data=line) + return result = requests.post(url, data=holmes_req.json()) result.raise_for_status() @@ -371,30 +454,25 @@ def holmes_chat(event: ExecutionBaseEvent, params: HolmesChatParams): if params.render_graph_images: try: for tool in holmes_result.tool_calls: - if tool.tool_name not in ["execute_prometheus_range_query", "query_datadog_metrics"]: + if tool.tool_name not in GRAPH_TOOLS: continue - holmes_result.analysis = re.sub(r"<<.*?>>", "", holmes_result.analysis).strip() + + # removes all embedded tool calls in this case as its not supported. + holmes_result.analysis = re.sub( + r"<<.*?>>", "", holmes_result.analysis + ).strip() json_content = json.loads(tool.result["data"]) - query_result = PrometheusQueryResult(data=json_content.get("data", {})) - try: - output_type_str = json_content.get("output_type", "Plain") - output_type = ChartValuesFormat[output_type_str] - except KeyError: - output_type = ChartValuesFormat.Plain # fallback in case of an invalid string - - chart = build_chart_from_prometheus_result( - query_result, json_content.get("description", "graph"), values_format=output_type - ) - contents = convert_svg_to_png(chart.render()) - name = json_content.get("description", "graph").replace(" ", "_") + contents, name = get_png_from_graph_tool(json_content) holmes_result.files.append(FileBlock(f"{name}.png", contents)) holmes_result.tool_calls = [ - tool for tool in holmes_result.tool_calls if tool.tool_name != "execute_prometheus_range_query" + tool + for tool in holmes_result.tool_calls + if tool.tool_name not in GRAPH_TOOLS ] except Exception: - logging.exception(f"Failed to convert tools to images") + logging.exception("Failed to convert tools to images") finding = Finding( title="AI Ask Chat", @@ -407,21 +485,29 @@ def holmes_chat(event: ExecutionBaseEvent, params: HolmesChatParams): ) finding.add_enrichment( - [HolmesChatResultsBlock(holmes_result=holmes_result)], enrichment_type=EnrichmentType.ai_analysis + [HolmesChatResultsBlock(holmes_result=holmes_result)], + enrichment_type=EnrichmentType.ai_analysis, ) event.add_finding(finding) except Exception as e: - logging.exception(f"Failed to get holmes chat for {cluster_name} cluster", exc_info=True) + logging.exception( + f"Failed to get holmes chat for {cluster_name} cluster", exc_info=True + ) handle_holmes_error(e) @action -def holmes_workload_chat(event: ExecutionBaseEvent, params: HolmesWorkloadHealthChatParams): +def holmes_workload_chat( + event: ExecutionBaseEvent, params: HolmesWorkloadHealthChatParams +): holmes_url = HolmesDiscovery.find_holmes_url(params.holmes_url) if not holmes_url: - raise ActionException(ErrorCodes.HOLMES_DISCOVERY_FAILED, "Robusta couldn't connect to the Holmes client.") + raise ActionException( + ErrorCodes.HOLMES_DISCOVERY_FAILED, + "Robusta couldn't connect to the Holmes client.", + ) try: holmes_req = HolmesWorkloadHealthRequest( @@ -431,7 +517,9 @@ def holmes_workload_chat(event: ExecutionBaseEvent, params: HolmesWorkloadHealth resource=params.resource, model=params.model, ) - result = requests.post(f"{holmes_url}/api/workload_health_chat", data=holmes_req.json()) + result = requests.post( + f"{holmes_url}/api/workload_health_chat", data=holmes_req.json() + ) result.raise_for_status() holmes_result = HolmesChatResult(**json.loads(result.text)) @@ -454,11 +542,81 @@ def holmes_workload_chat(event: ExecutionBaseEvent, params: HolmesWorkloadHealth failure=False, ) finding.add_enrichment( - [HolmesChatResultsBlock(holmes_result=holmes_result)], enrichment_type=EnrichmentType.ai_analysis + [HolmesChatResultsBlock(holmes_result=holmes_result)], + enrichment_type=EnrichmentType.ai_analysis, ) event.add_finding(finding) except Exception as e: - logging.exception(f"Failed to get holmes chat for health check of {params.resource}", exc_info=True) + logging.exception( + f"Failed to get holmes chat for health check of {params.resource}", + exc_info=True, + ) handle_holmes_error(e) + + +def stream_and_render_graphs(url, holmes_req, event): + with requests.post( + url, + data=holmes_req.json(), + stream=True, + headers={"Connection": "keep-alive"}, + ) as resp: + resp.raise_for_status() + for stream_event in resp.iter_content( + chunk_size=None, decode_unicode=True + ): # Avoid streaming chunks from holmes. send them as they arrive. + if stream_event: + event_lines = stream_event.splitlines() + # extra saftey check before proccessing event. + if len(event_lines) < 2: + event.ws(data=stream_event) + continue + + event_type = parse_sse_event_type(event_lines[0]) + # removes all embedded tool calls in this case as its not supported. + if event_type == StreamEvents.ANSWER_END.value: + stream_event = re.sub(r"<<.*?>>", "", stream_event) + + if event_type != StreamEvents.TOOL_RESULT.value: + event.ws(data=stream_event) + continue + + tool_res = parse_sse_data(event_lines[1]) + if not tool_res or tool_res.get("name", "") not in GRAPH_TOOLS: + event.ws(data=stream_event) + continue + + try: # convert graph tool to png. + tool_data = json.loads(tool_res["result"]["data"]) + content, _ = get_png_from_graph_tool(tool_data) + tool_res["result"]["data"] = base64.b64encode(content).decode( + "utf-8" + ) + tool_res["result_type"] = "png" + event.ws(data=create_sse_message(event_type, tool_res)) + except Exception: + logging.exception( + "Failed to convert graph tool to png %s,", event_lines[1] + ) + event.ws(data=stream_event) + + +def get_png_from_graph_tool(tool_result_data: dict): + query_result = PrometheusQueryResult(data=tool_result_data.get("data", {})) + try: + output_type_str = tool_result_data.get("output_type", "Plain") + output_type = ChartValuesFormat[output_type_str] + except KeyError: + output_type = ChartValuesFormat.Plain # fallback in case of an invalid string + + chart = build_chart_from_prometheus_result( + query_result, + tool_result_data.get("description", "graph"), + values_format=output_type, + ) + + contents = convert_svg_to_png(chart.render()) + name = tool_result_data.get("description", "graph").replace(" ", "_") + return contents, name diff --git a/src/robusta/core/stream/utils.py b/src/robusta/core/stream/utils.py new file mode 100644 index 000000000..4624efbc2 --- /dev/null +++ b/src/robusta/core/stream/utils.py @@ -0,0 +1,37 @@ +import json +from enum import Enum + + +def parse_sse_event_type(line: str): + """Parse SSE line and return event type or None""" + line = line.strip() + if line.startswith("event: "): + event_type = line[7:].strip() + return event_type + return None + + +def parse_sse_data(line: str): + """Parse SSE data line and return parsed JSON or None""" + if line.startswith("data: "): + try: + data = json.loads(line[6:].strip()) + return data + except json.JSONDecodeError: + return None + return None + + +def create_sse_message(event_type: str, data: dict): + return f"event: {event_type}\ndata: {json.dumps(data)}\n\n" + + +class StreamEvents(str, Enum): + ANSWER_END = "ai_answer_end" + START_TOOL = "start_tool_calling" + TOOL_RESULT = "tool_calling_result" + ERROR = "error" + AI_MESSAGE = "ai_message" + APPROVAL_REQUIRED = "approval_required" + TOKEN_COUNT = "token_count" + CONVERSATION_HISTORY_COMPACTED = "conversation_history_compacted" diff --git a/tests/test_ai_integration.py b/tests/test_ai_integration.py new file mode 100644 index 000000000..1f5b59261 --- /dev/null +++ b/tests/test_ai_integration.py @@ -0,0 +1,191 @@ +import base64 +import json +import pytest +from unittest.mock import Mock, patch +from robusta.core.model.base_params import HolmesChatParams +from robusta.core.model.events import ExecutionBaseEvent +from robusta.core.playbooks.internal.ai_integration import holmes_chat +from robusta.core.stream.utils import create_sse_message, StreamEvents + + +def parse_sse_message(sse_message: str): + """Parse an SSE message and return (event_type, data_dict).""" + lines = sse_message.strip().split("\n") + event_type = None + data = None + for line in lines: + if line.startswith("event: "): + event_type = line[7:].strip() + elif line.startswith("data: "): + data = json.loads(line[6:].strip()) + return event_type, data + + +class MockResponse: + """Mock response object for requests.post that supports streaming SSE events.""" + + def __init__(self, sse_events): + self.sse_events = sse_events + self.status_code = 200 + + def raise_for_status(self): + """Mock raise_for_status method.""" + pass + + def iter_content(self, chunk_size=None, decode_unicode=True): + """Generator that yields SSE events.""" + for event in self.sse_events: + yield event + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + pass + + +@pytest.fixture +def mock_event(): + event = Mock(spec=ExecutionBaseEvent) + event.ws = Mock() + event.get_context = Mock() + event.get_context.return_value.cluster_name = "test-cluster" + event.add_finding = Mock() + return event + + +@pytest.fixture +def holmes_chat_params(): + return HolmesChatParams( + ask="What is the status of the deployment?", + conversation_history=[ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi, how can I help?"}, + ], + model="gpt-4", + stream=True, + render_graph_images=True, + holmes_url="http://test-holmes:8080", + additional_system_prompt="Be concise", + enable_tool_approval=False, + ) + + +kubectl_tool = { + "tool_call_id": "tooluse_RYyDzKL1TWevnv3JvfhIww", + "role": "tool", + "description": "Count Kubernetes Resources: kubectl get pod --all-namespaces -o json | jq -c -r '.items[] | .metadata.name'", + "name": "kubernetes_count", + "result": { + "schema_version": "robusta:v1.0.0", + "status": "success", + "error": None, + "return_code": 0, + "data": "Command executed: kubectl get pod --all-namespaces -o json | jq -c -r '.items[] | .metadata.name'\n---\n 60 results\n---\nA *preview* of results is shown below (up to 10 results, up to 200 chars):\n 1\talertmanager-robusta-kube-prometheus-st-alertmanager-0\n 2\tcrashpod-77c67656c-qkfkn\n 3\tkrr-job-1954ac48-203f-4dbb-bfdf-49402d12bdc2-glb58\n 4\tkrr-job-51f7553b-fcad-4510-ba02-1a87a52c2315-nv4q7\n 5\tkrr-job-792188c7-c8fe-4f48-b6bc-4bb8d60717a8-gd4dj\n 6\tnginx-deployment-d556bf558-7x644\n 7\tpopeye-job-7982e9ab-fb66-464f-b4a2-a6b7ed2adb73-f99r2\n 8\tpopeye-job-d71cdd0c-92eb-488e-bc69-42f46e5a54d0-xs6mh\n 9\tprometheus-robusta-kube-prometheus-st-prometheus-0\n 10\trobusta-forwarder-764f79bf5b-lc9jl", + "url": None, + "invocation": 'echo "Command executed: kubectl get pod --all-namespaces -o json | jq -c -r \'.items[] | .metadata.name\'"\necho "---"\n\n# Execute the command and capture both stdout and stderr separately\ntemp_error=$(mktemp)\nmatches=$(kubectl get pod --all-namespaces -o json 2>"$temp_error" | jq -c -r \'.items[] | .metadata.name\' 2>>"$temp_error")\nexit_code=$?\nerror_output=$(cat "$temp_error")\nrm -f "$temp_error"\n\nif [ $exit_code -ne 0 ]; then\n echo "Error executing command (exit code: $exit_code):"\n echo "$error_output"\n exit $exit_code\nelse\n # Show any stderr warnings even if command succeeded\n if [ -n "$error_output" ]; then\n echo "Warnings/stderr output:"\n echo "$error_output"\n echo "---"\n fi\n\n # Filter out empty lines for accurate count\n filtered_matches=$(echo "$matches" | grep -v \'^$\' | grep -v \'^null$\')\n if [ -z "$filtered_matches" ]; then\n count=0\n else\n count=$(echo "$filtered_matches" | wc -l)\n fi\n preview=$(echo "$filtered_matches" | head -n 10 | cut -c 1-200 | nl)\n\n echo "$count results"\n echo "---"\n echo "A *preview* of results is shown below (up to 10 results, up to 200 chars):"\n echo "$preview"\nfi', + "params": {"kind": "pod", "jq_expr": ".items[] | .metadata.name"}, + "icon_url": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRPKA-U9m5BxYQDF1O7atMfj9EMMXEoGu4t0Q&s", + }, +} + +datadog_tool = { + "tool_call_id": "tooluse_3OxcweoOQLO_47WVTeC-SA", + "role": "tool", + "description": "Datadog: Query Metrics (Node CPU User Usage - Last 1 Hour)", + "name": "query_datadog_metrics", + "result": { + "schema_version": "robusta:v1.0.0", + "status": "success", + "error": None, + "return_code": None, + "data": '{\n "status": "success",\n "error_message": null,\n "random_key": "L7C5",\n "tool_name": "query_datadog_metrics",\n "description": "Node CPU User Usage - Last 1 Hour",\n "query": "system.cpu.user{*}",\n "start": "2025-11-25T14:20:17Z",\n "end": "2025-11-25T15:20:17Z",\n "step": 60,\n "output_type": "Percentage",\n "data": {\n "resultType": "matrix",\n "result": [\n {\n "metric": {\n "__name__": "system.cpu.user"\n },\n "values": [\n [\n 1764073220,\n "11.497647009320822"\n ],\n [\n 1764073240,\n "11.897418415931417"\n ],\n [\n 1764073260,\n "9.177484175000657"\n ],\n [\n 1764073280,\n "11.175071378170848"\n ],\n [\n 1764073300,\n "12.202757292755923"\n ],\n [\n 1764073320,\n "9.062731836502573"\n ],\n [\n 1764073340,\n "9.722311310908049"\n ],\n [\n 1764073360,\n "11.593790681526121"\n ],\n [\n 1764073380,\n "10.428452881489429"\n ],\n [\n 1764073400,\n "10.681149656416325"\n ],\n [\n 1764073420,\n "10.47686595847024"\n ],\n [\n 1764073440,\n "9.610235038738763"\n ],\n [\n 1764073460,\n "10.731352498588684"\n ],\n [\n 1764073480,\n "11.30864196049825"\n ],\n [\n 1764073500,\n "8.95367143881992"\n ],\n [\n 1764073520,\n "11.408187294645037"\n ],\n [\n 1764073540,\n "12.24438763133969"\n ],\n [\n 1764073560,\n "8.917150327916081"\n ],\n [\n 1764073580,\n "10.339755437099642"\n ],\n [\n 1764073600,\n "11.180332319689022"\n ],\n [\n 1764073620,\n "8.521326812524533"\n ],\n [\n 1764073640,\n "9.397463094015981"\n ],\n [\n 1764073660,\n "11.403082216172402"\n ],\n [\n 1764073680,\n "8.600455490298819"\n ],\n [\n 1764073700,\n "10.200019449056425"\n ],\n [\n 1764073720,\n "11.07380438386148"\n ],\n [\n 1764073740,\n "8.957280556745003"\n ],\n [\n 1764073760,\n "10.260519337517616"\n ],\n [\n 1764073780,\n "10.605474166284452"\n ],\n [\n 1764073800,\n "9.832253066469825"\n ],\n [\n 1764073820,\n "11.450528065300315"\n ],\n [\n 1764073840,\n "12.329729479693667"\n ],\n [\n 1764073860,\n "9.762210566175382"\n ],\n [\n 1764073880,\n "11.186269012690216"\n ],\n [\n 1764073900,\n "12.387762832688527"\n ],\n [\n 1764073920,\n "9.46121643546766"\n ],\n [\n 1764073940,\n "11.886144916545174"\n ],\n [\n 1764073960,\n "14.004224473259777"\n ],\n [\n 1764073980,\n "11.392103013044766"\n ],\n [\n 1764074000,\n "11.391088561604821"\n ],\n [\n 1764074020,\n "12.949367972034178"\n ],\n [\n 1764074040,\n "9.807194470374986"\n ],\n [\n 1764074060,\n "11.9628171017544"\n ],\n [\n 1764074080,\n "12.596095092738256"\n ],\n [\n 1764074100,\n "9.391677471103712"\n ],\n [\n 1764074120,\n "15.010615283380583"\n ],\n [\n 1764074140,\n "14.305756425296916"\n ],\n [\n 1764074160,\n "9.491673360664109"\n ],\n [\n 1764074180,\n "10.998333928961953"\n ],\n [\n 1764074200,\n "12.501552835674715"\n ],\n [\n 1764074220,\n "10.182627539279315"\n ],\n [\n 1764074240,\n "11.048783013407771"\n ],\n [\n 1764074260,\n "13.938976717606625"\n ],\n [\n 1764074280,\n "9.828927853994351"\n ],\n [\n 1764074300,\n "11.769693177648128"\n ],\n [\n 1764074320,\n "13.418180365912656"\n ],\n [\n 1764074340,\n "10.941499061439549"\n ],\n [\n 1764074360,\n "11.678202751909172"\n ],\n [\n 1764074380,\n "12.864131323972904"\n ],\n [\n 1764074400,\n "10.098404895032758"\n ],\n [\n 1764074420,\n "10.841631664129078"\n ],\n [\n 1764074440,\n "12.245070197472547"\n ],\n [\n 1764074460,\n "9.736263722153652"\n ],\n [\n 1764074480,\n "11.834225047820578"\n ],\n [\n 1764074500,\n "12.556133608274026"\n ],\n [\n 1764074520,\n "9.36438792195062"\n ],\n [\n 1764074540,\n "11.320352861403645"\n ],\n [\n 1764074560,\n "12.846953803765825"\n ],\n [\n 1764074580,\n "11.823618316557097"\n ],\n [\n 1764074600,\n "12.454640465465639"\n ],\n [\n 1764074620,\n "13.46102702067451"\n ],\n [\n 1764074640,\n "9.991321896145964"\n ],\n [\n 1764074660,\n "12.104817203340513"\n ],\n [\n 1764074680,\n "13.043451549102134"\n ],\n [\n 1764074700,\n "9.625698728873003"\n ],\n [\n 1764074720,\n "13.735210852910248"\n ],\n [\n 1764074740,\n "13.779471780408324"\n ],\n [\n 1764074760,\n "10.34351068612088"\n ],\n [\n 1764074780,\n "11.06237778275796"\n ],\n [\n 1764074800,\n "12.661974587097044"\n ],\n [\n 1764074820,\n "9.759910477177563"\n ],\n [\n 1764074840,\n "11.204254942591325"\n ],\n [\n 1764074860,\n "13.554092257345548"\n ],\n [\n 1764074880,\n "9.549400010644908"\n ],\n [\n 1764074900,\n "10.443284806259138"\n ],\n [\n 1764074920,\n "12.808593218950353"\n ],\n [\n 1764074940,\n "10.930731907797615"\n ],\n [\n 1764074960,\n "12.251668348428119"\n ],\n [\n 1764074980,\n "13.103989927048392"\n ],\n [\n 1764075000,\n "9.985083206070348"\n ],\n [\n 1764075020,\n "11.891199092884083"\n ],\n [\n 1764075040,\n "12.462970018050314"\n ],\n [\n 1764075060,\n "9.718230214847756"\n ],\n [\n 1764075080,\n "11.16170477353891"\n ],\n [\n 1764075100,\n "12.663882235458157"\n ],\n [\n 1764075120,\n "10.399801981495825"\n ],\n [\n 1764075140,\n "10.51365485425497"\n ],\n [\n 1764075160,\n "14.74760417635991"\n ],\n [\n 1764075180,\n "11.606312394523608"\n ],\n [\n 1764075200,\n "11.621289707075901"\n ],\n [\n 1764075220,\n "14.472888508127753"\n ],\n [\n 1764075240,\n "10.978381181157777"\n ],\n [\n 1764075260,\n "12.060160098915187"\n ],\n [\n 1764075280,\n "12.507441214843265"\n ],\n [\n 1764075300,\n "10.238384749632273"\n ],\n [\n 1764075320,\n "13.253770953832264"\n ],\n [\n 1764075340,\n "13.702760768520003"\n ],\n [\n 1764075360,\n "9.830476061994032"\n ],\n [\n 1764075380,\n "11.590948933947098"\n ],\n [\n 1764075400,\n "12.724135216627479"\n ],\n [\n 1764075420,\n "9.511661633673034"\n ],\n [\n 1764075440,\n "11.25374630915105"\n ],\n [\n 1764075460,\n "12.677301386882469"\n ],\n [\n 1764075480,\n "9.815140777759407"\n ],\n [\n 1764075500,\n "10.805061774716233"\n ],\n [\n 1764075520,\n "12.851178219205337"\n ],\n [\n 1764075540,\n "10.620350666907628"\n ],\n [\n 1764075560,\n "11.642024277774501"\n ],\n [\n 1764075580,\n "13.232491613128422"\n ],\n [\n 1764075600,\n "11.696719909480805"\n ],\n [\n 1764075620,\n "10.847754267278104"\n ],\n [\n 1764075640,\n "13.314911583607127"\n ],\n [\n 1764075660,\n "10.325759293513283"\n ],\n [\n 1764075680,\n "11.022537151184247"\n ],\n [\n 1764075700,\n "12.629090460110532"\n ],\n [\n 1764075720,\n "10.44479923179421"\n ],\n [\n 1764075740,\n "10.649959239635844"\n ],\n [\n 1764075760,\n "14.518618173177725"\n ],\n [\n 1764075780,\n "11.321412720223007"\n ],\n [\n 1764075800,\n "11.774820460161452"\n ],\n [\n 1764075820,\n "13.812555095442358"\n ],\n [\n 1764075840,\n "10.120145437597797"\n ],\n [\n 1764075860,\n "12.046317881845969"\n ],\n [\n 1764075880,\n "12.805767410233997"\n ],\n [\n 1764075900,\n "9.6656784649928"\n ],\n [\n 1764075920,\n "12.632129639418604"\n ],\n [\n 1764075940,\n "14.393283134249696"\n ],\n [\n 1764075960,\n "10.161545297749916"\n ],\n [\n 1764075980,\n "10.984265455486906"\n ],\n [\n 1764076000,\n "12.236982677242818"\n ],\n [\n 1764076020,\n "10.008177819392065"\n ],\n [\n 1764076040,\n "11.361748161787961"\n ],\n [\n 1764076060,\n "13.714046014489487"\n ],\n [\n 1764076080,\n "9.88049537496135"\n ],\n [\n 1764076100,\n "11.114650458544261"\n ],\n [\n 1764076120,\n "13.758089357291116"\n ],\n [\n 1764076140,\n "11.090980273106616"\n ],\n [\n 1764076160,\n "11.985101011491318"\n ],\n [\n 1764076180,\n "12.340779333817993"\n ],\n [\n 1764076200,\n "10.224300978883845"\n ],\n [\n 1764076220,\n "11.260988888730475"\n ],\n [\n 1764076240,\n "13.019929154137657"\n ],\n [\n 1764076260,\n "10.030551006825107"\n ],\n [\n 1764076280,\n "10.567454146978967"\n ],\n [\n 1764076300,\n "13.27085013349342"\n ],\n [\n 1764076320,\n "10.088068272561728"\n ],\n [\n 1764076340,\n "10.657376705268884"\n ],\n [\n 1764076360,\n "14.634303718266878"\n ],\n [\n 1764076380,\n "11.266816005157606"\n ],\n [\n 1764076400,\n "11.629290737544746"\n ],\n [\n 1764076420,\n "13.500646420361416"\n ],\n [\n 1764076440,\n "10.305605115426058"\n ],\n [\n 1764076460,\n "10.98116267453516"\n ],\n [\n 1764076480,\n "13.559637845511247"\n ],\n [\n 1764076500,\n "12.62638236735979"\n ],\n [\n 1764076520,\n "15.33249331452554"\n ],\n [\n 1764076540,\n "13.982126527177082"\n ],\n [\n 1764076560,\n "9.996950128990497"\n ],\n [\n 1764076580,\n "12.018939098109572"\n ],\n [\n 1764076600,\n "12.507007093969078"\n ],\n [\n 1764076620,\n "9.527177087266514"\n ],\n [\n 1764076640,\n "11.410318956641675"\n ],\n [\n 1764076660,\n "14.274524213529821"\n ],\n [\n 1764076680,\n "9.823602374863695"\n ],\n [\n 1764076700,\n "10.603017880372224"\n ],\n [\n 1764076720,\n "12.244592244047748"\n ],\n [\n 1764076740,\n "10.49303030909029"\n ],\n [\n 1764076760,\n "11.401758774529249"\n ],\n [\n 1764076780,\n "12.254855812737176"\n ],\n [\n 1764076800,\n "11.514570607099847"\n ]\n ]\n }\n ]\n }\n}', + "url": None, + "invocation": None, + "params": { + "query": "system.cpu.user{*}", + "description": "Node CPU User Usage - Last 1 Hour", + "from_time": "-3600", + "output_type": "Percentage", + }, + "icon_url": "https://imgix.datadoghq.com//img/about/presskit/DDlogo.jpg", + }, +} + + +prom_tool = { + "tool_call_id": "tooluse_wzkFIucoQBCDTh2R0K-Aow", + "role": "tool", + "description": "Prometheus: Query (Node memory used over the last hour)", + "name": "execute_prometheus_range_query", + "result": { + "schema_version": "robusta:v1.0.0", + "status": "success", + "error": None, + "return_code": None, + "data": '{\n "status": "success",\n "error_message": null,\n "data": {\n "resultType": "matrix",\n "result": [\n {\n "metric": {\n "container": "node-exporter",\n "endpoint": "http-metrics",\n "instance": "10.224.0.4:9104",\n "job": "node-exporter",\n "namespace": "default",\n "pod": "robusta-prometheus-node-exporter-6wpxv",\n "service": "robusta-prometheus-node-exporter"\n },\n "values": [\n [\n 1764072332,\n "3572977664"\n ],\n [\n 1764072392,\n "3537756160"\n ],\n [\n 1764072452,\n "3545972736"\n ],\n [\n 1764072512,\n "3544150016"\n ],\n [\n 1764072572,\n "3533594624"\n ],\n [\n 1764072632,\n "3554119680"\n ],\n [\n 1764072692,\n "3554951168"\n ],\n [\n 1764072752,\n "3519492096"\n ],\n [\n 1764072812,\n "3540918272"\n ],\n [\n 1764072872,\n "3544043520"\n ],\n [\n 1764072932,\n "3558350848"\n ],\n [\n 1764072992,\n "3552022528"\n ],\n [\n 1764073052,\n "3565129728"\n ],\n [\n 1764073112,\n "3535163392"\n ],\n [\n 1764073172,\n "3578920960"\n ],\n [\n 1764073232,\n "3543629824"\n ],\n [\n 1764073292,\n "3550691328"\n ],\n [\n 1764073352,\n "3560235008"\n ],\n [\n 1764073412,\n "3555373056"\n ],\n [\n 1764073472,\n "3570520064"\n ],\n [\n 1764073532,\n "3574845440"\n ],\n [\n 1764073592,\n "3539652608"\n ],\n [\n 1764073652,\n "3569573888"\n ],\n [\n 1764073712,\n "3580882944"\n ],\n [\n 1764073772,\n "3602333696"\n ],\n [\n 1764073832,\n "3616030720"\n ],\n [\n 1764073892,\n "3608498176"\n ],\n [\n 1764073952,\n "3610931200"\n ],\n [\n 1764074012,\n "3606700032"\n ],\n [\n 1764074072,\n "3616079872"\n ],\n [\n 1764074132,\n "3614482432"\n ],\n [\n 1764074192,\n "3613757440"\n ],\n [\n 1764074252,\n "3587878912"\n ],\n [\n 1764074312,\n "3598036992"\n ],\n [\n 1764074372,\n "3616669696"\n ],\n [\n 1764074432,\n "3594452992"\n ],\n [\n 1764074492,\n "3601604608"\n ],\n [\n 1764074552,\n "3606708224"\n ],\n [\n 1764074612,\n "3617087488"\n ],\n [\n 1764074672,\n "3610701824"\n ],\n [\n 1764074732,\n "3621089280"\n ],\n [\n 1764074792,\n "3627216896"\n ],\n [\n 1764074852,\n "3626217472"\n ],\n [\n 1764074912,\n "3622838272"\n ],\n [\n 1764074972,\n "3613995008"\n ],\n [\n 1764075032,\n "3606310912"\n ],\n [\n 1764075092,\n "3634552832"\n ],\n [\n 1764075152,\n "3618246656"\n ],\n [\n 1764075212,\n "3601166336"\n ],\n [\n 1764075272,\n "3569119232"\n ],\n [\n 1764075332,\n "3566489600"\n ],\n [\n 1764075392,\n "3572002816"\n ],\n [\n 1764075452,\n "3608752128"\n ],\n [\n 1764075512,\n "3610542080"\n ],\n [\n 1764075572,\n "3636908032"\n ],\n [\n 1764075632,\n "3640434688"\n ],\n [\n 1764075692,\n "3626635264"\n ],\n [\n 1764075752,\n "3589390336"\n ],\n [\n 1764075812,\n "3583930368"\n ],\n [\n 1764075872,\n "3558903808"\n ],\n [\n 1764075932,\n "3556708352"\n ]\n ]\n },\n {\n "metric": {\n "container": "node-exporter",\n "endpoint": "http-metrics",\n "instance": "10.224.0.5:9104",\n "job": "node-exporter",\n "namespace": "default",\n "pod": "robusta-prometheus-node-exporter-vrptl",\n "service": "robusta-prometheus-node-exporter"\n },\n "values": [\n [\n 1764072332,\n "3119882240"\n ],\n [\n 1764072392,\n "3127107584"\n ],\n [\n 1764072452,\n "3114704896"\n ],\n [\n 1764072512,\n "3098738688"\n ],\n [\n 1764072572,\n "3096498176"\n ],\n [\n 1764072632,\n "3099578368"\n ],\n [\n 1764072692,\n "3093700608"\n ],\n [\n 1764072752,\n "3132846080"\n ],\n [\n 1764072812,\n "3128274944"\n ],\n [\n 1764072872,\n "3111079936"\n ],\n [\n 1764072932,\n "3111907328"\n ],\n [\n 1764072992,\n "3103399936"\n ],\n [\n 1764073052,\n "3106709504"\n ],\n [\n 1764073112,\n "3102035968"\n ],\n [\n 1764073172,\n "3106697216"\n ],\n [\n 1764073232,\n "3148988416"\n ],\n [\n 1764073292,\n "3110178816"\n ],\n [\n 1764073352,\n "3127742464"\n ],\n [\n 1764073412,\n "3119374336"\n ],\n [\n 1764073472,\n "3135569920"\n ],\n [\n 1764073532,\n "3135770624"\n ],\n [\n 1764073592,\n "3134320640"\n ],\n [\n 1764073652,\n "3138416640"\n ],\n [\n 1764073712,\n "3134271488"\n ],\n [\n 1764073772,\n "3128188928"\n ],\n [\n 1764073832,\n "3127025664"\n ],\n [\n 1764073892,\n "3126128640"\n ],\n [\n 1764073952,\n "3148656640"\n ],\n [\n 1764074012,\n "3141668864"\n ],\n [\n 1764074072,\n "3130101760"\n ],\n [\n 1764074132,\n "3126611968"\n ],\n [\n 1764074192,\n "3125719040"\n ],\n [\n 1764074252,\n "3126050816"\n ],\n [\n 1764074312,\n "3127148544"\n ],\n [\n 1764074372,\n "3122876416"\n ],\n [\n 1764074432,\n "3118923776"\n ],\n [\n 1764074492,\n "3117084672"\n ],\n [\n 1764074552,\n "3138912256"\n ],\n [\n 1764074612,\n "3138990080"\n ],\n [\n 1764074672,\n "3128147968"\n ],\n [\n 1764074732,\n "3127156736"\n ],\n [\n 1764074792,\n "3131760640"\n ],\n [\n 1764074852,\n "3148009472"\n ],\n [\n 1764074912,\n "3134013440"\n ],\n [\n 1764074972,\n "3137740800"\n ],\n [\n 1764075032,\n "3140558848"\n ],\n [\n 1764075092,\n "3147956224"\n ],\n [\n 1764075152,\n "3136434176"\n ],\n [\n 1764075212,\n "3159691264"\n ],\n [\n 1764075272,\n "3135582208"\n ],\n [\n 1764075332,\n "3128713216"\n ],\n [\n 1764075392,\n "3120377856"\n ],\n [\n 1764075452,\n "3119050752"\n ],\n [\n 1764075512,\n "3113914368"\n ],\n [\n 1764075572,\n "3105837056"\n ],\n [\n 1764075632,\n "3106181120"\n ],\n [\n 1764075692,\n "3099713536"\n ],\n [\n 1764075752,\n "3103080448"\n ],\n [\n 1764075812,\n "3120189440"\n ],\n [\n 1764075872,\n "3118223360"\n ],\n [\n 1764075932,\n "3128819712"\n ]\n ]\n },\n {\n "metric": {\n "container": "node-exporter",\n "endpoint": "http-metrics",\n "instance": "10.224.0.8:9104",\n "job": "node-exporter",\n "namespace": "default",\n "pod": "robusta-prometheus-node-exporter-4xswz",\n "service": "robusta-prometheus-node-exporter"\n },\n "values": [\n [\n 1764072332,\n "1550110720"\n ],\n [\n 1764072392,\n "1556418560"\n ],\n [\n 1764072452,\n "1552818176"\n ],\n [\n 1764072512,\n "1568550912"\n ],\n [\n 1764072572,\n "1570553856"\n ],\n [\n 1764072632,\n "1565925376"\n ],\n [\n 1764072692,\n "1558519808"\n ],\n [\n 1764072752,\n "1565147136"\n ],\n [\n 1764072812,\n "1556869120"\n ],\n [\n 1764072872,\n "1564893184"\n ],\n [\n 1764072932,\n "1561124864"\n ],\n [\n 1764072992,\n "1558581248"\n ],\n [\n 1764073052,\n "1553575936"\n ],\n [\n 1764073112,\n "1555468288"\n ],\n [\n 1764073172,\n "1560424448"\n ],\n [\n 1764073232,\n "1563721728"\n ],\n [\n 1764073292,\n "1565892608"\n ],\n [\n 1764073352,\n "1569091584"\n ],\n [\n 1764073412,\n "1560940544"\n ],\n [\n 1764073472,\n "1558835200"\n ],\n [\n 1764073532,\n "1564901376"\n ],\n [\n 1764073592,\n "1557291008"\n ],\n [\n 1764073652,\n "1557823488"\n ],\n [\n 1764073712,\n "1564626944"\n ],\n [\n 1764073772,\n "1554653184"\n ],\n [\n 1764073832,\n "1565917184"\n ],\n [\n 1764073892,\n "1564991488"\n ],\n [\n 1764073952,\n "1564946432"\n ],\n [\n 1764074012,\n "1550614528"\n ],\n [\n 1764074072,\n "1554907136"\n ],\n [\n 1764074132,\n "1567526912"\n ],\n [\n 1764074192,\n "1558114304"\n ],\n [\n 1764074252,\n "1560743936"\n ],\n [\n 1764074312,\n "1562632192"\n ],\n [\n 1764074372,\n "1570713600"\n ],\n [\n 1764074432,\n "1562075136"\n ],\n [\n 1764074492,\n "1578500096"\n ],\n [\n 1764074552,\n "1577680896"\n ],\n [\n 1764074612,\n "1563664384"\n ],\n [\n 1764074672,\n "1573056512"\n ],\n [\n 1764074732,\n "1555251200"\n ],\n [\n 1764074792,\n "1563787264"\n ],\n [\n 1764074852,\n "1568706560"\n ],\n [\n 1764074912,\n "1571561472"\n ],\n [\n 1764074972,\n "1564798976"\n ],\n [\n 1764075032,\n "1570070528"\n ],\n [\n 1764075092,\n "1564405760"\n ],\n [\n 1764075152,\n "1560776704"\n ],\n [\n 1764075212,\n "1569566720"\n ],\n [\n 1764075272,\n "1563283456"\n ],\n [\n 1764075332,\n "1560375296"\n ],\n [\n 1764075392,\n "1557499904"\n ],\n [\n 1764075452,\n "1565253632"\n ],\n [\n 1764075512,\n "1539444736"\n ],\n [\n 1764075572,\n "1565818880"\n ],\n [\n 1764075632,\n "1568919552"\n ],\n [\n 1764075692,\n "1561030656"\n ],\n [\n 1764075752,\n "1566998528"\n ],\n [\n 1764075812,\n "1561960448"\n ],\n [\n 1764075872,\n "1561812992"\n ],\n [\n 1764075932,\n "1569964032"\n ]\n ]\n }\n ]\n },\n "random_key": "95tC",\n "tool_name": "execute_prometheus_range_query",\n "description": "Node memory used over the last hour",\n "query": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes",\n "start": "2025-11-25T12:05:32Z",\n "end": "2025-11-25T13:05:32Z",\n "step": 60.0,\n "output_type": "Bytes",\n "data_summary": null\n}', + "url": None, + "invocation": None, + "params": { + "query": "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes", + "description": "Node memory used over the last hour", + "output_type": "Bytes", + "start": "-3600", + }, + "icon_url": "https://upload.wikimedia.org/wikipedia/commons/3/38/Prometheus_software_logo.svg", + }, +} + + +@patch("robusta.core.playbooks.internal.ai_integration.requests.post") +def test_holmes_chat_streaming_with_sse_events( + mock_post, mock_event, holmes_chat_params +): + """Test holmes_chat function with streaming SSE events.""" + + sse_events = [ + create_sse_message( + StreamEvents.START_TOOL.value, + { + "tool_name": "execute_prometheus_range_query", + "id": "tooluse_NXaZGTwuRKS1ogDEKP4M9Q", + }, + ), + create_sse_message(StreamEvents.TOOL_RESULT.value, kubectl_tool), + create_sse_message(StreamEvents.TOOL_RESULT.value, datadog_tool), + create_sse_message(StreamEvents.TOOL_RESULT.value, prom_tool), + create_sse_message( + StreamEvents.ANSWER_END.value, + { + "analysis": 'some analysis... add the << {"type": "datadogql", "tool_name": "query_datadog_metrics", "random_key": "U572"} >> rest of analysis', + }, + ), + ] + + mock_response = MockResponse(sse_events) + mock_post.return_value = mock_response + + holmes_chat(mock_event, holmes_chat_params) + mock_post.assert_called_once() + assert mock_event.ws.call_count == len(sse_events) + + # events should stay the same + assert mock_event.ws.call_args_list[0][1]["data"] == sse_events[0] + assert mock_event.ws.call_args_list[1][1]["data"] == sse_events[1] + # graph tools change + datadog_output = mock_event.ws.call_args_list[2][1]["data"] + _, data = parse_sse_message(datadog_output) + decoded = base64.b64decode(data["result"]["data"]) + assert decoded[:8] == b"\x89PNG\r\n\x1a\n", "Not a valid PNG" + + prom_output = mock_event.ws.call_args_list[3][1]["data"] + _, data = parse_sse_message(prom_output) + decoded = base64.b64decode(data["result"]["data"]) + assert decoded[:8] == b"\x89PNG\r\n\x1a\n", "Not a valid PNG" + # answer << >> parts is gone. + assert mock_event.ws.call_args_list[4][1]["data"] == create_sse_message( + StreamEvents.ANSWER_END.value, + { + "analysis": "some analysis... add the rest of analysis", + }, + )