diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index 9bceb71549..fc3a210f6f 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -33549,6 +33549,10 @@ components: description: The `add_env_vars` processor adds environment variable values to log events. properties: + enabled: + description: Whether this processor is enabled. + example: true + type: boolean id: description: The unique identifier for this component. Used to reference this processor in the pipeline. @@ -33559,14 +33563,6 @@ components: targets. example: service:my-service type: string - inputs: - description: A list of component IDs whose output is used as the input for - this processor. - example: - - datadog-agent-source - items: - type: string - type: array type: $ref: '#/components/schemas/ObservabilityPipelineAddEnvVarsProcessorType' variables: @@ -33578,8 +33574,8 @@ components: - id - type - include - - inputs - variables + - enabled type: object ObservabilityPipelineAddEnvVarsProcessorType: default: add_env_vars @@ -33608,6 +33604,10 @@ components: ObservabilityPipelineAddFieldsProcessor: description: The `add_fields` processor adds static key-value fields to logs. properties: + enabled: + description: Whether this processor is enabled. + example: true + type: boolean fields: description: A list of static fields (key-value pairs) that is added to each log event processed by this component. @@ -33625,14 +33625,6 @@ components: targets. example: service:my-service type: string - inputs: - description: A list of component IDs whose output is used as the `input` - for this component. - example: - - datadog-agent-source - items: - type: string - type: array type: $ref: '#/components/schemas/ObservabilityPipelineAddFieldsProcessorType' required: @@ -33640,7 +33632,7 @@ components: - type - include - fields - - inputs + - enabled type: object ObservabilityPipelineAddFieldsProcessorType: default: add_fields @@ -33943,15 +33935,20 @@ components: $ref: '#/components/schemas/ObservabilityPipelineConfigDestinationItem' type: array processors: - description: A list of processors that transform or enrich log data. + description: A list of processor groups that transform or enrich log data. example: - - id: filter-processor + - enabled: true + id: my-processor-group include: service:my-service inputs: - datadog-agent-source - type: filter + processors: + - enabled: true + id: filter-processor + include: status:error + type: filter items: - $ref: '#/components/schemas/ObservabilityPipelineConfigProcessorItem' + $ref: '#/components/schemas/ObservabilityPipelineConfigProcessorGroup' type: array sources: description: A list of configured data sources for the pipeline. @@ -33987,6 +33984,68 @@ components: - $ref: '#/components/schemas/ObservabilityPipelineAmazonSecurityLakeDestination' - $ref: '#/components/schemas/ObservabilityPipelineCrowdStrikeNextGenSiemDestination' - $ref: '#/components/schemas/ObservabilityPipelineGooglePubSubDestination' + ObservabilityPipelineConfigProcessorGroup: + description: A group of processors. + example: + id: my-processor-group + include: service:my-service + inputs: + - datadog-agent-source + processors: + - fields: + - name: env + value: prod + id: add-fields-processor + include: '*' + type: add_fields + - id: filter-processor + include: status:error + type: filter + properties: + enabled: + description: Whether this processor group is enabled. + example: true + type: boolean + id: + description: The unique identifier for the processor group. + example: grouped-processors + type: string + include: + description: Conditional expression for when this processor group should + execute. + example: service:my-service + type: string + inputs: + description: A list of component IDs whose output is used as the input for + this processor group. + example: + - datadog-agent-source + items: + type: string + type: array + processors: + description: Processors applied sequentially within this group. Events flow + through each processor in order. + example: + - fields: + - name: env + value: prod + id: add-fields-processor + include: '*' + type: add_fields + - id: filter-processor + include: status:error + type: filter + items: + $ref: '#/components/schemas/ObservabilityPipelineConfigProcessorItem' + type: array + required: + - id + - include + - inputs + - processors + - enabled + type: object ObservabilityPipelineConfigProcessorItem: description: A processor for the pipeline. oneOf: @@ -34104,6 +34163,10 @@ components: Remap Language (VRL)](https://vector.dev/docs/reference/vrl/) scripts with advanced filtering capabilities. properties: + enabled: + description: Whether this processor is enabled. + example: true + type: boolean id: description: The unique identifier for this processor. example: remap-vrl-processor @@ -34115,14 +34178,6 @@ components: processor. example: '*' type: string - inputs: - description: A list of component IDs whose output is used as the input for - this processor. - example: - - datadog-agent-source - items: - type: string - type: array remaps: description: Array of VRL remap rules. items: @@ -34136,7 +34191,7 @@ components: - type - include - remaps - - inputs + - enabled type: object ObservabilityPipelineCustomProcessorRemap: description: Defines a single VRL remap rule with its own filtering and transformation @@ -34167,7 +34222,6 @@ components: - include - name - source - - enabled - drop_on_error type: object ObservabilityPipelineCustomProcessorType: @@ -34276,6 +34330,10 @@ components: properties: action: $ref: '#/components/schemas/ObservabilityPipelineDatadogTagsProcessorAction' + enabled: + description: Whether this processor is enabled. + example: true + type: boolean id: description: The unique identifier for this component. Used to reference this component in other parts of the pipeline (for example, as the `input` @@ -34287,14 +34345,6 @@ components: targets. example: service:my-service type: string - inputs: - description: A list of component IDs whose output is used as the `input` - for this component. - example: - - datadog-agent-source - items: - type: string - type: array keys: description: A list of tag keys. example: @@ -34315,7 +34365,7 @@ components: - mode - action - keys - - inputs + - enabled type: object ObservabilityPipelineDatadogTagsProcessorAction: description: The action to take on tags with matching keys. @@ -34361,6 +34411,10 @@ components: ObservabilityPipelineDedupeProcessor: description: The `dedupe` processor removes duplicate fields in log events. properties: + enabled: + description: Whether this processor is enabled. + example: true + type: boolean fields: description: A list of log field paths to check for duplicates. example: @@ -34378,14 +34432,6 @@ components: targets. example: service:my-service type: string - inputs: - description: A list of component IDs whose output is used as the input for - this processor. - example: - - parse-json-processor - items: - type: string - type: array mode: $ref: '#/components/schemas/ObservabilityPipelineDedupeProcessorMode' type: @@ -34394,9 +34440,9 @@ components: - id - type - include - - inputs - fields - mode + - enabled type: object ObservabilityPipelineDedupeProcessorMode: description: The deduplication mode to apply to the fields. @@ -34602,6 +34648,10 @@ components: description: The `enrichment_table` processor enriches logs using a static CSV file or GeoIP database. properties: + enabled: + description: Whether this processor is enabled. + example: true + type: boolean file: $ref: '#/components/schemas/ObservabilityPipelineEnrichmentTableFile' geoip: @@ -34615,14 +34665,6 @@ components: targets. example: source:my-source type: string - inputs: - description: A list of component IDs whose output is used as the input for - this processor. - example: - - add-fields-processor - items: - type: string - type: array target: description: Path where enrichment results should be stored in the log. example: enriched.geoip @@ -34633,8 +34675,8 @@ components: - id - type - include - - inputs - target + - enabled type: object ObservabilityPipelineEnrichmentTableProcessorType: default: enrichment_table @@ -34665,6 +34707,10 @@ components: on a Datadog search query. Logs that match the `include` query are passed through; others are discarded. properties: + enabled: + description: Whether this processor is enabled. + example: true + type: boolean id: description: The unique identifier for this component. Used to reference this component in other parts of the pipeline (for example, as the `input` @@ -34677,21 +34723,13 @@ components: components; others are dropped. example: service:my-service type: string - inputs: - description: A list of component IDs whose output is used as the `input` - for this component. - example: - - datadog-agent-source - items: - type: string - type: array type: $ref: '#/components/schemas/ObservabilityPipelineFilterProcessorType' required: - id - type - include - - inputs + - enabled type: object ObservabilityPipelineFilterProcessorType: default: filter @@ -34771,6 +34809,10 @@ components: Metrics can be counters, gauges, or distributions and optionally grouped by log fields.' properties: + enabled: + description: Whether this processor is enabled. + example: true + type: boolean id: description: The unique identifier for this component. Used to reference this component in other parts of the pipeline. @@ -34781,14 +34823,6 @@ components: targets. example: service:my-service type: string - inputs: - description: A list of component IDs whose output is used as the `input` - for this processor. - example: - - source-id - items: - type: string - type: array metrics: description: Configuration for generating individual metrics. items: @@ -34799,9 +34833,7 @@ components: required: - id - type - - inputs - - include - - metrics + - enabled type: object ObservabilityPipelineGenerateMetricsProcessorType: default: generate_datadog_metrics @@ -35398,6 +35430,10 @@ components: description: The `ocsf_mapper` processor transforms logs into the OCSF schema using a predefined mapping configuration. properties: + enabled: + description: Whether this processor is enabled. + example: true + type: boolean id: description: The unique identifier for this component. Used to reference this component in other parts of the pipeline. @@ -35408,14 +35444,6 @@ components: targets. example: service:my-service type: string - inputs: - description: A list of component IDs whose output is used as the `input` - for this processor. - example: - - filter-processor - items: - type: string - type: array mappings: description: A list of mapping rules to convert events to the OCSF format. items: @@ -35427,8 +35455,8 @@ components: - id - type - include - - inputs - mappings + - enabled type: object ObservabilityPipelineOcsfMapperProcessorMapping: description: Defines how specific events are transformed to OCSF using a mapping @@ -35533,6 +35561,10 @@ components: by Datadog. example: true type: boolean + enabled: + description: Whether this processor is enabled. + example: true + type: boolean id: description: A unique identifier for this processor. example: parse-grok-processor @@ -35542,14 +35574,6 @@ components: targets. example: service:my-service type: string - inputs: - description: A list of component IDs whose output is used as the `input` - for this component. - example: - - datadog-agent-source - items: - type: string - type: array rules: description: The list of Grok parsing rules. If multiple matching rules are provided, they are evaluated in order. The first successful match @@ -35563,8 +35587,8 @@ components: - id - type - include - - inputs - rules + - enabled type: object ObservabilityPipelineParseGrokProcessorRule: description: 'A Grok parsing rule used in the `parse_grok` processor. Each rule @@ -35649,6 +35673,10 @@ components: and flattens it into the event. This is useful when logs contain embedded JSON as a string. properties: + enabled: + description: Whether this processor is enabled. + example: true + type: boolean field: description: The name of the log field that contains a JSON string. example: message @@ -35664,14 +35692,6 @@ components: targets. example: service:my-service type: string - inputs: - description: A list of component IDs whose output is used as the `input` - for this component. - example: - - datadog-agent-source - items: - type: string - type: array type: $ref: '#/components/schemas/ObservabilityPipelineParseJSONProcessorType' required: @@ -35679,7 +35699,7 @@ components: - type - include - field - - inputs + - enabled type: object ObservabilityPipelineParseJSONProcessorType: default: parse_json @@ -35712,6 +35732,10 @@ components: the filter query continue through the pipeline. example: false type: boolean + enabled: + description: Whether this processor is enabled. + example: true + type: boolean id: description: The unique identifier for this component. Used to reference this component in other parts of the pipeline (for example, as the `input` @@ -35727,14 +35751,6 @@ components: targets. example: service:my-service type: string - inputs: - description: A list of component IDs whose output is used as the `input` - for this component. - example: - - datadog-agent-source - items: - type: string - type: array limit: $ref: '#/components/schemas/ObservabilityPipelineQuotaProcessorLimit' name: @@ -35764,9 +35780,8 @@ components: - type - include - name - - drop_events - limit - - inputs + - enabled type: object ObservabilityPipelineQuotaProcessorLimit: description: The maximum amount of data or number of events allowed before the @@ -35842,6 +35857,10 @@ components: description: The `reduce` processor aggregates and merges logs based on matching keys and merge strategies. properties: + enabled: + description: Whether this processor is enabled. + example: true + type: boolean group_by: description: A list of fields used to group log events for merging. example: @@ -35859,14 +35878,6 @@ components: targets. example: env:prod type: string - inputs: - description: A list of component IDs whose output is used as the input for - this processor. - example: - - parse-json-processor - items: - type: string - type: array merge_strategies: description: List of merge strategies defining how values from grouped events should be combined. @@ -35879,9 +35890,9 @@ components: - id - type - include - - inputs - group_by - merge_strategies + - enabled type: object ObservabilityPipelineReduceProcessorMergeStrategy: description: Defines how a specific field should be merged across grouped events. @@ -35938,6 +35949,10 @@ components: ObservabilityPipelineRemoveFieldsProcessor: description: The `remove_fields` processor deletes specified fields from logs. properties: + enabled: + description: Whether this processor is enabled. + example: true + type: boolean fields: description: A list of field names to be removed from each log event. example: @@ -35957,13 +35972,6 @@ components: targets. example: service:my-service type: string - inputs: - description: The `PipelineRemoveFieldsProcessor` `inputs`. - example: - - datadog-agent-source - items: - type: string - type: array type: $ref: '#/components/schemas/ObservabilityPipelineRemoveFieldsProcessorType' required: @@ -35971,7 +35979,7 @@ components: - type - include - fields - - inputs + - enabled type: object ObservabilityPipelineRemoveFieldsProcessorType: default: remove_fields @@ -35985,6 +35993,10 @@ components: ObservabilityPipelineRenameFieldsProcessor: description: The `rename_fields` processor changes field names. properties: + enabled: + description: Whether this processor is enabled. + example: true + type: boolean fields: description: A list of rename rules specifying which fields to rename in the event, what to rename them to, and whether to preserve the original @@ -36003,14 +36015,6 @@ components: targets. example: service:my-service type: string - inputs: - description: A list of component IDs whose output is used as the `input` - for this component. - example: - - datadog-agent-source - items: - type: string - type: array type: $ref: '#/components/schemas/ObservabilityPipelineRenameFieldsProcessorType' required: @@ -36018,7 +36022,7 @@ components: - type - include - fields - - inputs + - enabled type: object ObservabilityPipelineRenameFieldsProcessorField: description: Defines how to rename a field in log events. @@ -36124,6 +36128,10 @@ components: description: The `sample` processor allows probabilistic sampling of logs at a fixed rate. properties: + enabled: + description: Whether this processor is enabled. + example: true + type: boolean id: description: The unique identifier for this component. Used to reference this component in other parts of the pipeline (for example, as the `input` @@ -36135,14 +36143,6 @@ components: targets. example: service:my-service type: string - inputs: - description: A list of component IDs whose output is used as the `input` - for this component. - example: - - datadog-agent-source - items: - type: string - type: array percentage: description: The percentage of logs to sample. example: 10.0 @@ -36160,7 +36160,7 @@ components: - id - type - include - - inputs + - enabled type: object ObservabilityPipelineSampleProcessorType: default: sample @@ -36175,6 +36175,10 @@ components: description: The `sensitive_data_scanner` processor detects and optionally redacts sensitive data in log events. properties: + enabled: + description: Whether this processor is enabled. + example: true + type: boolean id: description: The unique identifier for this component. Used to reference this component in other parts of the pipeline (e.g., as input to downstream @@ -36186,14 +36190,6 @@ components: targets. example: source:prod type: string - inputs: - description: A list of component IDs whose output is used as the `input` - for this component. - example: - - parse-json-processor - items: - type: string - type: array rules: description: A list of rules for identifying and acting on sensitive data patterns. @@ -36206,8 +36202,8 @@ components: - id - type - include - - inputs - rules + - enabled type: object ObservabilityPipelineSensitiveDataScannerProcessorAction: description: Defines what action to take when sensitive data is matched. @@ -37168,6 +37164,10 @@ components: description: The `throttle` processor limits the number of events that pass through over a given time window. properties: + enabled: + description: Whether this processor is enabled. + example: true + type: boolean group_by: description: Optional list of fields used to group events before the threshold has been reached. @@ -37185,14 +37185,6 @@ components: targets. example: env:prod type: string - inputs: - description: A list of component IDs whose output is used as the input for - this processor. - example: - - datadog-agent-source - items: - type: string - type: array threshold: description: the number of events allowed in a given time window. Events sent after the threshold has been reached, are dropped. @@ -37210,9 +37202,9 @@ components: - id - type - include - - inputs - threshold - window + - enabled type: object ObservabilityPipelineThrottleProcessorType: default: throttle diff --git a/docs/datadog_api_client.v2.model.rst b/docs/datadog_api_client.v2.model.rst index 60fdfd49e2..5ed62fd0e6 100644 --- a/docs/datadog_api_client.v2.model.rst +++ b/docs/datadog_api_client.v2.model.rst @@ -14823,6 +14823,13 @@ datadog\_api\_client.v2.model.observability\_pipeline\_config\_destination\_item :members: :show-inheritance: +datadog\_api\_client.v2.model.observability\_pipeline\_config\_processor\_group module +-------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.observability_pipeline_config_processor_group + :members: + :show-inheritance: + datadog\_api\_client.v2.model.observability\_pipeline\_config\_processor\_item module ------------------------------------------------------------------------------------- diff --git a/examples/v2/observability-pipelines/CreatePipeline.py b/examples/v2/observability-pipelines/CreatePipeline.py index a6303525e7..87b50b26c0 100644 --- a/examples/v2/observability-pipelines/CreatePipeline.py +++ b/examples/v2/observability-pipelines/CreatePipeline.py @@ -5,6 +5,9 @@ from datadog_api_client import ApiClient, Configuration from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig +from datadog_api_client.v2.model.observability_pipeline_config_processor_group import ( + ObservabilityPipelineConfigProcessorGroup, +) from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import ( ObservabilityPipelineDatadogAgentSource, @@ -33,19 +36,27 @@ ObservabilityPipelineDatadogLogsDestination( id="datadog-logs-destination", inputs=[ - "filter-processor", + "my-processor-group", ], type=ObservabilityPipelineDatadogLogsDestinationType.DATADOG_LOGS, ), ], processors=[ - ObservabilityPipelineFilterProcessor( - id="filter-processor", + ObservabilityPipelineConfigProcessorGroup( + enabled=True, + id="my-processor-group", include="service:my-service", inputs=[ "datadog-agent-source", ], - type=ObservabilityPipelineFilterProcessorType.FILTER, + processors=[ + ObservabilityPipelineFilterProcessor( + enabled=True, + id="filter-processor", + include="status:error", + type=ObservabilityPipelineFilterProcessorType.FILTER, + ), + ], ), ], sources=[ diff --git a/examples/v2/observability-pipelines/UpdatePipeline.py b/examples/v2/observability-pipelines/UpdatePipeline.py index 9d036b867c..c8e05195fa 100644 --- a/examples/v2/observability-pipelines/UpdatePipeline.py +++ b/examples/v2/observability-pipelines/UpdatePipeline.py @@ -7,6 +7,9 @@ from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi from datadog_api_client.v2.model.observability_pipeline import ObservabilityPipeline from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig +from datadog_api_client.v2.model.observability_pipeline_config_processor_group import ( + ObservabilityPipelineConfigProcessorGroup, +) from datadog_api_client.v2.model.observability_pipeline_data import ObservabilityPipelineData from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import ( @@ -37,19 +40,27 @@ ObservabilityPipelineDatadogLogsDestination( id="updated-datadog-logs-destination-id", inputs=[ - "filter-processor", + "my-processor-group", ], type=ObservabilityPipelineDatadogLogsDestinationType.DATADOG_LOGS, ), ], processors=[ - ObservabilityPipelineFilterProcessor( - id="filter-processor", + ObservabilityPipelineConfigProcessorGroup( + enabled=True, + id="my-processor-group", include="service:my-service", inputs=[ "datadog-agent-source", ], - type=ObservabilityPipelineFilterProcessorType.FILTER, + processors=[ + ObservabilityPipelineFilterProcessor( + enabled=True, + id="filter-processor", + include="status:error", + type=ObservabilityPipelineFilterProcessorType.FILTER, + ), + ], ), ], sources=[ diff --git a/examples/v2/observability-pipelines/ValidatePipeline.py b/examples/v2/observability-pipelines/ValidatePipeline.py index 31cd6775b3..a55d4a6368 100644 --- a/examples/v2/observability-pipelines/ValidatePipeline.py +++ b/examples/v2/observability-pipelines/ValidatePipeline.py @@ -5,6 +5,9 @@ from datadog_api_client import ApiClient, Configuration from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig +from datadog_api_client.v2.model.observability_pipeline_config_processor_group import ( + ObservabilityPipelineConfigProcessorGroup, +) from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import ( ObservabilityPipelineDatadogAgentSource, @@ -33,19 +36,27 @@ ObservabilityPipelineDatadogLogsDestination( id="datadog-logs-destination", inputs=[ - "filter-processor", + "my-processor-group", ], type=ObservabilityPipelineDatadogLogsDestinationType.DATADOG_LOGS, ), ], processors=[ - ObservabilityPipelineFilterProcessor( - id="filter-processor", + ObservabilityPipelineConfigProcessorGroup( + enabled=True, + id="my-processor-group", include="service:my-service", inputs=[ "datadog-agent-source", ], - type=ObservabilityPipelineFilterProcessorType.FILTER, + processors=[ + ObservabilityPipelineFilterProcessor( + enabled=True, + id="filter-processor", + include="status:error", + type=ObservabilityPipelineFilterProcessorType.FILTER, + ), + ], ), ], sources=[ diff --git a/src/datadog_api_client/v2/model/observability_pipeline_add_env_vars_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_add_env_vars_processor.py index ab11ebe286..879b74ebce 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_add_env_vars_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_add_env_vars_processor.py @@ -31,26 +31,26 @@ def openapi_types(_): ) return { + "enabled": (bool,), "id": (str,), "include": (str,), - "inputs": ([str],), "type": (ObservabilityPipelineAddEnvVarsProcessorType,), "variables": ([ObservabilityPipelineAddEnvVarsProcessorVariable],), } attribute_map = { + "enabled": "enabled", "id": "id", "include": "include", - "inputs": "inputs", "type": "type", "variables": "variables", } def __init__( self_, + enabled: bool, id: str, include: str, - inputs: List[str], type: ObservabilityPipelineAddEnvVarsProcessorType, variables: List[ObservabilityPipelineAddEnvVarsProcessorVariable], **kwargs, @@ -58,15 +58,15 @@ def __init__( """ The ``add_env_vars`` processor adds environment variable values to log events. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param id: The unique identifier for this component. Used to reference this processor in the pipeline. :type id: str :param include: A Datadog search query used to determine which logs this processor targets. :type include: str - :param inputs: A list of component IDs whose output is used as the input for this processor. - :type inputs: [str] - :param type: The processor type. The value should always be ``add_env_vars``. :type type: ObservabilityPipelineAddEnvVarsProcessorType @@ -75,8 +75,8 @@ def __init__( """ super().__init__(kwargs) + self_.enabled = enabled self_.id = id self_.include = include - self_.inputs = inputs self_.type = type self_.variables = variables diff --git a/src/datadog_api_client/v2/model/observability_pipeline_add_fields_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_add_fields_processor.py index 023b7a6d57..229c27128d 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_add_fields_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_add_fields_processor.py @@ -27,33 +27,36 @@ def openapi_types(_): ) return { + "enabled": (bool,), "fields": ([ObservabilityPipelineFieldValue],), "id": (str,), "include": (str,), - "inputs": ([str],), "type": (ObservabilityPipelineAddFieldsProcessorType,), } attribute_map = { + "enabled": "enabled", "fields": "fields", "id": "id", "include": "include", - "inputs": "inputs", "type": "type", } def __init__( self_, + enabled: bool, fields: List[ObservabilityPipelineFieldValue], id: str, include: str, - inputs: List[str], type: ObservabilityPipelineAddFieldsProcessorType, **kwargs, ): """ The ``add_fields`` processor adds static key-value fields to logs. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param fields: A list of static fields (key-value pairs) that is added to each log event processed by this component. :type fields: [ObservabilityPipelineFieldValue] @@ -63,16 +66,13 @@ def __init__( :param include: A Datadog search query used to determine which logs this processor targets. :type include: str - :param inputs: A list of component IDs whose output is used as the ``input`` for this component. - :type inputs: [str] - :param type: The processor type. The value should always be ``add_fields``. :type type: ObservabilityPipelineAddFieldsProcessorType """ super().__init__(kwargs) + self_.enabled = enabled self_.fields = fields self_.id = id self_.include = include - self_.inputs = inputs self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_config.py b/src/datadog_api_client/v2/model/observability_pipeline_config.py index e612d38a2c..48cf842a5d 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_config.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_config.py @@ -17,8 +17,8 @@ from datadog_api_client.v2.model.observability_pipeline_config_destination_item import ( ObservabilityPipelineConfigDestinationItem, ) - from datadog_api_client.v2.model.observability_pipeline_config_processor_item import ( - ObservabilityPipelineConfigProcessorItem, + from datadog_api_client.v2.model.observability_pipeline_config_processor_group import ( + ObservabilityPipelineConfigProcessorGroup, ) from datadog_api_client.v2.model.observability_pipeline_config_source_item import ( ObservabilityPipelineConfigSourceItem, @@ -76,48 +76,6 @@ from datadog_api_client.v2.model.observability_pipeline_google_pub_sub_destination import ( ObservabilityPipelineGooglePubSubDestination, ) - from datadog_api_client.v2.model.observability_pipeline_filter_processor import ObservabilityPipelineFilterProcessor - from datadog_api_client.v2.model.observability_pipeline_parse_json_processor import ( - ObservabilityPipelineParseJSONProcessor, - ) - from datadog_api_client.v2.model.observability_pipeline_quota_processor import ObservabilityPipelineQuotaProcessor - from datadog_api_client.v2.model.observability_pipeline_add_fields_processor import ( - ObservabilityPipelineAddFieldsProcessor, - ) - from datadog_api_client.v2.model.observability_pipeline_remove_fields_processor import ( - ObservabilityPipelineRemoveFieldsProcessor, - ) - from datadog_api_client.v2.model.observability_pipeline_rename_fields_processor import ( - ObservabilityPipelineRenameFieldsProcessor, - ) - from datadog_api_client.v2.model.observability_pipeline_generate_metrics_processor import ( - ObservabilityPipelineGenerateMetricsProcessor, - ) - from datadog_api_client.v2.model.observability_pipeline_sample_processor import ObservabilityPipelineSampleProcessor - from datadog_api_client.v2.model.observability_pipeline_parse_grok_processor import ( - ObservabilityPipelineParseGrokProcessor, - ) - from datadog_api_client.v2.model.observability_pipeline_sensitive_data_scanner_processor import ( - ObservabilityPipelineSensitiveDataScannerProcessor, - ) - from datadog_api_client.v2.model.observability_pipeline_ocsf_mapper_processor import ( - ObservabilityPipelineOcsfMapperProcessor, - ) - from datadog_api_client.v2.model.observability_pipeline_add_env_vars_processor import ( - ObservabilityPipelineAddEnvVarsProcessor, - ) - from datadog_api_client.v2.model.observability_pipeline_dedupe_processor import ObservabilityPipelineDedupeProcessor - from datadog_api_client.v2.model.observability_pipeline_enrichment_table_processor import ( - ObservabilityPipelineEnrichmentTableProcessor, - ) - from datadog_api_client.v2.model.observability_pipeline_reduce_processor import ObservabilityPipelineReduceProcessor - from datadog_api_client.v2.model.observability_pipeline_throttle_processor import ( - ObservabilityPipelineThrottleProcessor, - ) - from datadog_api_client.v2.model.observability_pipeline_custom_processor import ObservabilityPipelineCustomProcessor - from datadog_api_client.v2.model.observability_pipeline_datadog_tags_processor import ( - ObservabilityPipelineDatadogTagsProcessor, - ) from datadog_api_client.v2.model.observability_pipeline_kafka_source import ObservabilityPipelineKafkaSource from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import ( ObservabilityPipelineDatadogAgentSource, @@ -160,8 +118,8 @@ def openapi_types(_): from datadog_api_client.v2.model.observability_pipeline_config_destination_item import ( ObservabilityPipelineConfigDestinationItem, ) - from datadog_api_client.v2.model.observability_pipeline_config_processor_item import ( - ObservabilityPipelineConfigProcessorItem, + from datadog_api_client.v2.model.observability_pipeline_config_processor_group import ( + ObservabilityPipelineConfigProcessorGroup, ) from datadog_api_client.v2.model.observability_pipeline_config_source_item import ( ObservabilityPipelineConfigSourceItem, @@ -169,7 +127,7 @@ def openapi_types(_): return { "destinations": ([ObservabilityPipelineConfigDestinationItem],), - "processors": ([ObservabilityPipelineConfigProcessorItem],), + "processors": ([ObservabilityPipelineConfigProcessorGroup],), "sources": ([ObservabilityPipelineConfigSourceItem],), } @@ -226,32 +184,7 @@ def __init__( ObservabilityPipelineSocketSource, ] ], - processors: Union[ - List[ - Union[ - ObservabilityPipelineConfigProcessorItem, - ObservabilityPipelineFilterProcessor, - ObservabilityPipelineParseJSONProcessor, - ObservabilityPipelineQuotaProcessor, - ObservabilityPipelineAddFieldsProcessor, - ObservabilityPipelineRemoveFieldsProcessor, - ObservabilityPipelineRenameFieldsProcessor, - ObservabilityPipelineGenerateMetricsProcessor, - ObservabilityPipelineSampleProcessor, - ObservabilityPipelineParseGrokProcessor, - ObservabilityPipelineSensitiveDataScannerProcessor, - ObservabilityPipelineOcsfMapperProcessor, - ObservabilityPipelineAddEnvVarsProcessor, - ObservabilityPipelineDedupeProcessor, - ObservabilityPipelineEnrichmentTableProcessor, - ObservabilityPipelineReduceProcessor, - ObservabilityPipelineThrottleProcessor, - ObservabilityPipelineCustomProcessor, - ObservabilityPipelineDatadogTagsProcessor, - ] - ], - UnsetType, - ] = unset, + processors: Union[List[ObservabilityPipelineConfigProcessorGroup], UnsetType] = unset, **kwargs, ): """ @@ -260,8 +193,8 @@ def __init__( :param destinations: A list of destination components where processed logs are sent. :type destinations: [ObservabilityPipelineConfigDestinationItem] - :param processors: A list of processors that transform or enrich log data. - :type processors: [ObservabilityPipelineConfigProcessorItem], optional + :param processors: A list of processor groups that transform or enrich log data. + :type processors: [ObservabilityPipelineConfigProcessorGroup], optional :param sources: A list of configured data sources for the pipeline. :type sources: [ObservabilityPipelineConfigSourceItem] diff --git a/src/datadog_api_client/v2/model/observability_pipeline_config_processor_group.py b/src/datadog_api_client/v2/model/observability_pipeline_config_processor_group.py new file mode 100644 index 0000000000..badfa9c4bb --- /dev/null +++ b/src/datadog_api_client/v2/model/observability_pipeline_config_processor_group.py @@ -0,0 +1,140 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import List, Union, TYPE_CHECKING + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +if TYPE_CHECKING: + from datadog_api_client.v2.model.observability_pipeline_config_processor_item import ( + ObservabilityPipelineConfigProcessorItem, + ) + from datadog_api_client.v2.model.observability_pipeline_filter_processor import ObservabilityPipelineFilterProcessor + from datadog_api_client.v2.model.observability_pipeline_parse_json_processor import ( + ObservabilityPipelineParseJSONProcessor, + ) + from datadog_api_client.v2.model.observability_pipeline_quota_processor import ObservabilityPipelineQuotaProcessor + from datadog_api_client.v2.model.observability_pipeline_add_fields_processor import ( + ObservabilityPipelineAddFieldsProcessor, + ) + from datadog_api_client.v2.model.observability_pipeline_remove_fields_processor import ( + ObservabilityPipelineRemoveFieldsProcessor, + ) + from datadog_api_client.v2.model.observability_pipeline_rename_fields_processor import ( + ObservabilityPipelineRenameFieldsProcessor, + ) + from datadog_api_client.v2.model.observability_pipeline_generate_metrics_processor import ( + ObservabilityPipelineGenerateMetricsProcessor, + ) + from datadog_api_client.v2.model.observability_pipeline_sample_processor import ObservabilityPipelineSampleProcessor + from datadog_api_client.v2.model.observability_pipeline_parse_grok_processor import ( + ObservabilityPipelineParseGrokProcessor, + ) + from datadog_api_client.v2.model.observability_pipeline_sensitive_data_scanner_processor import ( + ObservabilityPipelineSensitiveDataScannerProcessor, + ) + from datadog_api_client.v2.model.observability_pipeline_ocsf_mapper_processor import ( + ObservabilityPipelineOcsfMapperProcessor, + ) + from datadog_api_client.v2.model.observability_pipeline_add_env_vars_processor import ( + ObservabilityPipelineAddEnvVarsProcessor, + ) + from datadog_api_client.v2.model.observability_pipeline_dedupe_processor import ObservabilityPipelineDedupeProcessor + from datadog_api_client.v2.model.observability_pipeline_enrichment_table_processor import ( + ObservabilityPipelineEnrichmentTableProcessor, + ) + from datadog_api_client.v2.model.observability_pipeline_reduce_processor import ObservabilityPipelineReduceProcessor + from datadog_api_client.v2.model.observability_pipeline_throttle_processor import ( + ObservabilityPipelineThrottleProcessor, + ) + from datadog_api_client.v2.model.observability_pipeline_custom_processor import ObservabilityPipelineCustomProcessor + from datadog_api_client.v2.model.observability_pipeline_datadog_tags_processor import ( + ObservabilityPipelineDatadogTagsProcessor, + ) + + +class ObservabilityPipelineConfigProcessorGroup(ModelNormal): + @cached_property + def openapi_types(_): + from datadog_api_client.v2.model.observability_pipeline_config_processor_item import ( + ObservabilityPipelineConfigProcessorItem, + ) + + return { + "enabled": (bool,), + "id": (str,), + "include": (str,), + "inputs": ([str],), + "processors": ([ObservabilityPipelineConfigProcessorItem],), + } + + attribute_map = { + "enabled": "enabled", + "id": "id", + "include": "include", + "inputs": "inputs", + "processors": "processors", + } + + def __init__( + self_, + enabled: bool, + id: str, + include: str, + inputs: List[str], + processors: List[ + Union[ + ObservabilityPipelineConfigProcessorItem, + ObservabilityPipelineFilterProcessor, + ObservabilityPipelineParseJSONProcessor, + ObservabilityPipelineQuotaProcessor, + ObservabilityPipelineAddFieldsProcessor, + ObservabilityPipelineRemoveFieldsProcessor, + ObservabilityPipelineRenameFieldsProcessor, + ObservabilityPipelineGenerateMetricsProcessor, + ObservabilityPipelineSampleProcessor, + ObservabilityPipelineParseGrokProcessor, + ObservabilityPipelineSensitiveDataScannerProcessor, + ObservabilityPipelineOcsfMapperProcessor, + ObservabilityPipelineAddEnvVarsProcessor, + ObservabilityPipelineDedupeProcessor, + ObservabilityPipelineEnrichmentTableProcessor, + ObservabilityPipelineReduceProcessor, + ObservabilityPipelineThrottleProcessor, + ObservabilityPipelineCustomProcessor, + ObservabilityPipelineDatadogTagsProcessor, + ] + ], + **kwargs, + ): + """ + A group of processors. + + :param enabled: Whether this processor group is enabled. + :type enabled: bool + + :param id: The unique identifier for the processor group. + :type id: str + + :param include: Conditional expression for when this processor group should execute. + :type include: str + + :param inputs: A list of component IDs whose output is used as the input for this processor group. + :type inputs: [str] + + :param processors: Processors applied sequentially within this group. Events flow through each processor in order. + :type processors: [ObservabilityPipelineConfigProcessorItem] + """ + super().__init__(kwargs) + + self_.enabled = enabled + self_.id = id + self_.include = include + self_.inputs = inputs + self_.processors = processors diff --git a/src/datadog_api_client/v2/model/observability_pipeline_config_processor_item.py b/src/datadog_api_client/v2/model/observability_pipeline_config_processor_item.py index 7c57096d1a..fc1d8480e7 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_config_processor_item.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_config_processor_item.py @@ -15,15 +15,15 @@ def __init__(self, **kwargs): """ A processor for the pipeline. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param id: The unique identifier for this component. Used to reference this component in other parts of the pipeline (for example, as the `input` to downstream components). :type id: str :param include: A Datadog search query used to determine which logs should pass through the filter. Logs that match this query continue to downstream components; others are dropped. :type include: str - :param inputs: A list of component IDs whose output is used as the `input` for this component. - :type inputs: [str] - :param type: The processor type. The value should always be `filter`. :type type: ObservabilityPipelineFilterProcessorType @@ -31,7 +31,7 @@ def __init__(self, **kwargs): :type field: str :param drop_events: If set to `true`, logs that matched the quota filter and sent after the quota has been met are dropped; only logs that did not match the filter query continue through the pipeline. - :type drop_events: bool + :type drop_events: bool, optional :param ignore_when_missing_partitions: If `true`, the processor skips quota checks when partition fields are missing from the logs. :type ignore_when_missing_partitions: bool, optional @@ -58,7 +58,7 @@ def __init__(self, **kwargs): :type fields: [ObservabilityPipelineFieldValue] :param metrics: Configuration for generating individual metrics. - :type metrics: [ObservabilityPipelineGeneratedMetric] + :type metrics: [ObservabilityPipelineGeneratedMetric], optional :param percentage: The percentage of logs to sample. :type percentage: float, optional diff --git a/src/datadog_api_client/v2/model/observability_pipeline_custom_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_custom_processor.py index 214d511ebc..4219ae2d87 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_custom_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_custom_processor.py @@ -37,25 +37,25 @@ def openapi_types(_): ) return { + "enabled": (bool,), "id": (str,), "include": (str,), - "inputs": ([str],), "remaps": ([ObservabilityPipelineCustomProcessorRemap],), "type": (ObservabilityPipelineCustomProcessorType,), } attribute_map = { + "enabled": "enabled", "id": "id", "include": "include", - "inputs": "inputs", "remaps": "remaps", "type": "type", } def __init__( self_, + enabled: bool, id: str, - inputs: List[str], remaps: List[ObservabilityPipelineCustomProcessorRemap], type: ObservabilityPipelineCustomProcessorType, **kwargs, @@ -63,15 +63,15 @@ def __init__( """ The ``custom_processor`` processor transforms events using `Vector Remap Language (VRL) `_ scripts with advanced filtering capabilities. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param id: The unique identifier for this processor. :type id: str :param include: A Datadog search query used to determine which logs this processor targets. This field should always be set to ``*`` for the custom_processor processor. :type include: str - :param inputs: A list of component IDs whose output is used as the input for this processor. - :type inputs: [str] - :param remaps: Array of VRL remap rules. :type remaps: [ObservabilityPipelineCustomProcessorRemap] @@ -81,8 +81,8 @@ def __init__( super().__init__(kwargs) include = kwargs.get("include", "*") + self_.enabled = enabled self_.id = id self_.include = include - self_.inputs = inputs self_.remaps = remaps self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_custom_processor_remap.py b/src/datadog_api_client/v2/model/observability_pipeline_custom_processor_remap.py index ab063499f6..601b381d70 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_custom_processor_remap.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_custom_processor_remap.py @@ -3,10 +3,13 @@ # Copyright 2019-Present Datadog, Inc. from __future__ import annotations +from typing import Union from datadog_api_client.model_utils import ( ModelNormal, cached_property, + unset, + UnsetType, ) @@ -29,7 +32,15 @@ def openapi_types(_): "source": "source", } - def __init__(self_, drop_on_error: bool, enabled: bool, include: str, name: str, source: str, **kwargs): + def __init__( + self_, + drop_on_error: bool, + include: str, + name: str, + source: str, + enabled: Union[bool, UnsetType] = unset, + **kwargs, + ): """ Defines a single VRL remap rule with its own filtering and transformation logic. @@ -37,7 +48,7 @@ def __init__(self_, drop_on_error: bool, enabled: bool, include: str, name: str, :type drop_on_error: bool :param enabled: Whether this remap rule is enabled. - :type enabled: bool + :type enabled: bool, optional :param include: A Datadog search query used to filter events for this specific remap rule. :type include: str @@ -48,10 +59,11 @@ def __init__(self_, drop_on_error: bool, enabled: bool, include: str, name: str, :param source: The VRL script source code that defines the processing logic. :type source: str """ + if enabled is not unset: + kwargs["enabled"] = enabled super().__init__(kwargs) self_.drop_on_error = drop_on_error - self_.enabled = enabled self_.include = include self_.name = name self_.source = source diff --git a/src/datadog_api_client/v2/model/observability_pipeline_datadog_tags_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_datadog_tags_processor.py index 23cedc4d69..8864ee8c29 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_datadog_tags_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_datadog_tags_processor.py @@ -38,9 +38,9 @@ def openapi_types(_): return { "action": (ObservabilityPipelineDatadogTagsProcessorAction,), + "enabled": (bool,), "id": (str,), "include": (str,), - "inputs": ([str],), "keys": ([str],), "mode": (ObservabilityPipelineDatadogTagsProcessorMode,), "type": (ObservabilityPipelineDatadogTagsProcessorType,), @@ -48,9 +48,9 @@ def openapi_types(_): attribute_map = { "action": "action", + "enabled": "enabled", "id": "id", "include": "include", - "inputs": "inputs", "keys": "keys", "mode": "mode", "type": "type", @@ -59,9 +59,9 @@ def openapi_types(_): def __init__( self_, action: ObservabilityPipelineDatadogTagsProcessorAction, + enabled: bool, id: str, include: str, - inputs: List[str], keys: List[str], mode: ObservabilityPipelineDatadogTagsProcessorMode, type: ObservabilityPipelineDatadogTagsProcessorType, @@ -73,15 +73,15 @@ def __init__( :param action: The action to take on tags with matching keys. :type action: ObservabilityPipelineDatadogTagsProcessorAction + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param id: The unique identifier for this component. Used to reference this component in other parts of the pipeline (for example, as the ``input`` to downstream components). :type id: str :param include: A Datadog search query used to determine which logs this processor targets. :type include: str - :param inputs: A list of component IDs whose output is used as the ``input`` for this component. - :type inputs: [str] - :param keys: A list of tag keys. :type keys: [str] @@ -94,9 +94,9 @@ def __init__( super().__init__(kwargs) self_.action = action + self_.enabled = enabled self_.id = id self_.include = include - self_.inputs = inputs self_.keys = keys self_.mode = mode self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_dedupe_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_dedupe_processor.py index 9021940cfc..e4abb6a8ff 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_dedupe_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_dedupe_processor.py @@ -31,29 +31,29 @@ def openapi_types(_): ) return { + "enabled": (bool,), "fields": ([str],), "id": (str,), "include": (str,), - "inputs": ([str],), "mode": (ObservabilityPipelineDedupeProcessorMode,), "type": (ObservabilityPipelineDedupeProcessorType,), } attribute_map = { + "enabled": "enabled", "fields": "fields", "id": "id", "include": "include", - "inputs": "inputs", "mode": "mode", "type": "type", } def __init__( self_, + enabled: bool, fields: List[str], id: str, include: str, - inputs: List[str], mode: ObservabilityPipelineDedupeProcessorMode, type: ObservabilityPipelineDedupeProcessorType, **kwargs, @@ -61,6 +61,9 @@ def __init__( """ The ``dedupe`` processor removes duplicate fields in log events. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param fields: A list of log field paths to check for duplicates. :type fields: [str] @@ -70,9 +73,6 @@ def __init__( :param include: A Datadog search query used to determine which logs this processor targets. :type include: str - :param inputs: A list of component IDs whose output is used as the input for this processor. - :type inputs: [str] - :param mode: The deduplication mode to apply to the fields. :type mode: ObservabilityPipelineDedupeProcessorMode @@ -81,9 +81,9 @@ def __init__( """ super().__init__(kwargs) + self_.enabled = enabled self_.fields = fields self_.id = id self_.include = include - self_.inputs = inputs self_.mode = mode self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_enrichment_table_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_enrichment_table_processor.py index 480f484822..2bc18bd802 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_enrichment_table_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_enrichment_table_processor.py @@ -3,7 +3,7 @@ # Copyright 2019-Present Datadog, Inc. from __future__ import annotations -from typing import List, Union, TYPE_CHECKING +from typing import Union, TYPE_CHECKING from datadog_api_client.model_utils import ( ModelNormal, @@ -39,30 +39,30 @@ def openapi_types(_): ) return { + "enabled": (bool,), "file": (ObservabilityPipelineEnrichmentTableFile,), "geoip": (ObservabilityPipelineEnrichmentTableGeoIp,), "id": (str,), "include": (str,), - "inputs": ([str],), "target": (str,), "type": (ObservabilityPipelineEnrichmentTableProcessorType,), } attribute_map = { + "enabled": "enabled", "file": "file", "geoip": "geoip", "id": "id", "include": "include", - "inputs": "inputs", "target": "target", "type": "type", } def __init__( self_, + enabled: bool, id: str, include: str, - inputs: List[str], target: str, type: ObservabilityPipelineEnrichmentTableProcessorType, file: Union[ObservabilityPipelineEnrichmentTableFile, UnsetType] = unset, @@ -72,6 +72,9 @@ def __init__( """ The ``enrichment_table`` processor enriches logs using a static CSV file or GeoIP database. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param file: Defines a static enrichment table loaded from a CSV file. :type file: ObservabilityPipelineEnrichmentTableFile, optional @@ -84,9 +87,6 @@ def __init__( :param include: A Datadog search query used to determine which logs this processor targets. :type include: str - :param inputs: A list of component IDs whose output is used as the input for this processor. - :type inputs: [str] - :param target: Path where enrichment results should be stored in the log. :type target: str @@ -99,8 +99,8 @@ def __init__( kwargs["geoip"] = geoip super().__init__(kwargs) + self_.enabled = enabled self_.id = id self_.include = include - self_.inputs = inputs self_.target = target self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_filter_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_filter_processor.py index d7ada2fb81..30a2bfed16 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_filter_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_filter_processor.py @@ -3,7 +3,7 @@ # Copyright 2019-Present Datadog, Inc. from __future__ import annotations -from typing import List, TYPE_CHECKING +from typing import TYPE_CHECKING from datadog_api_client.model_utils import ( ModelNormal, @@ -25,40 +25,38 @@ def openapi_types(_): ) return { + "enabled": (bool,), "id": (str,), "include": (str,), - "inputs": ([str],), "type": (ObservabilityPipelineFilterProcessorType,), } attribute_map = { + "enabled": "enabled", "id": "id", "include": "include", - "inputs": "inputs", "type": "type", } - def __init__( - self_, id: str, include: str, inputs: List[str], type: ObservabilityPipelineFilterProcessorType, **kwargs - ): + def __init__(self_, enabled: bool, id: str, include: str, type: ObservabilityPipelineFilterProcessorType, **kwargs): """ The ``filter`` processor allows conditional processing of logs based on a Datadog search query. Logs that match the ``include`` query are passed through; others are discarded. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param id: The unique identifier for this component. Used to reference this component in other parts of the pipeline (for example, as the ``input`` to downstream components). :type id: str :param include: A Datadog search query used to determine which logs should pass through the filter. Logs that match this query continue to downstream components; others are dropped. :type include: str - :param inputs: A list of component IDs whose output is used as the ``input`` for this component. - :type inputs: [str] - :param type: The processor type. The value should always be ``filter``. :type type: ObservabilityPipelineFilterProcessorType """ super().__init__(kwargs) + self_.enabled = enabled self_.id = id self_.include = include - self_.inputs = inputs self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_generate_metrics_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_generate_metrics_processor.py index b0eee92908..058f6e5d68 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_generate_metrics_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_generate_metrics_processor.py @@ -3,11 +3,13 @@ # Copyright 2019-Present Datadog, Inc. from __future__ import annotations -from typing import List, TYPE_CHECKING +from typing import List, Union, TYPE_CHECKING from datadog_api_client.model_utils import ( ModelNormal, cached_property, + unset, + UnsetType, ) @@ -29,53 +31,55 @@ def openapi_types(_): ) return { + "enabled": (bool,), "id": (str,), "include": (str,), - "inputs": ([str],), "metrics": ([ObservabilityPipelineGeneratedMetric],), "type": (ObservabilityPipelineGenerateMetricsProcessorType,), } attribute_map = { + "enabled": "enabled", "id": "id", "include": "include", - "inputs": "inputs", "metrics": "metrics", "type": "type", } def __init__( self_, + enabled: bool, id: str, - include: str, - inputs: List[str], - metrics: List[ObservabilityPipelineGeneratedMetric], type: ObservabilityPipelineGenerateMetricsProcessorType, + include: Union[str, UnsetType] = unset, + metrics: Union[List[ObservabilityPipelineGeneratedMetric], UnsetType] = unset, **kwargs, ): """ The ``generate_datadog_metrics`` processor creates custom metrics from logs and sends them to Datadog. Metrics can be counters, gauges, or distributions and optionally grouped by log fields. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param id: The unique identifier for this component. Used to reference this component in other parts of the pipeline. :type id: str :param include: A Datadog search query used to determine which logs this processor targets. - :type include: str - - :param inputs: A list of component IDs whose output is used as the ``input`` for this processor. - :type inputs: [str] + :type include: str, optional :param metrics: Configuration for generating individual metrics. - :type metrics: [ObservabilityPipelineGeneratedMetric] + :type metrics: [ObservabilityPipelineGeneratedMetric], optional :param type: The processor type. Always ``generate_datadog_metrics``. :type type: ObservabilityPipelineGenerateMetricsProcessorType """ + if include is not unset: + kwargs["include"] = include + if metrics is not unset: + kwargs["metrics"] = metrics super().__init__(kwargs) + self_.enabled = enabled self_.id = id - self_.include = include - self_.inputs = inputs - self_.metrics = metrics self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_ocsf_mapper_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_ocsf_mapper_processor.py index 7917f1c81e..e49bde37ec 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_ocsf_mapper_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_ocsf_mapper_processor.py @@ -31,26 +31,26 @@ def openapi_types(_): ) return { + "enabled": (bool,), "id": (str,), "include": (str,), - "inputs": ([str],), "mappings": ([ObservabilityPipelineOcsfMapperProcessorMapping],), "type": (ObservabilityPipelineOcsfMapperProcessorType,), } attribute_map = { + "enabled": "enabled", "id": "id", "include": "include", - "inputs": "inputs", "mappings": "mappings", "type": "type", } def __init__( self_, + enabled: bool, id: str, include: str, - inputs: List[str], mappings: List[ObservabilityPipelineOcsfMapperProcessorMapping], type: ObservabilityPipelineOcsfMapperProcessorType, **kwargs, @@ -58,15 +58,15 @@ def __init__( """ The ``ocsf_mapper`` processor transforms logs into the OCSF schema using a predefined mapping configuration. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param id: The unique identifier for this component. Used to reference this component in other parts of the pipeline. :type id: str :param include: A Datadog search query used to determine which logs this processor targets. :type include: str - :param inputs: A list of component IDs whose output is used as the ``input`` for this processor. - :type inputs: [str] - :param mappings: A list of mapping rules to convert events to the OCSF format. :type mappings: [ObservabilityPipelineOcsfMapperProcessorMapping] @@ -75,8 +75,8 @@ def __init__( """ super().__init__(kwargs) + self_.enabled = enabled self_.id = id self_.include = include - self_.inputs = inputs self_.mappings = mappings self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_parse_grok_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_parse_grok_processor.py index c7b2f65cfc..a527670797 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_parse_grok_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_parse_grok_processor.py @@ -34,27 +34,27 @@ def openapi_types(_): return { "disable_library_rules": (bool,), + "enabled": (bool,), "id": (str,), "include": (str,), - "inputs": ([str],), "rules": ([ObservabilityPipelineParseGrokProcessorRule],), "type": (ObservabilityPipelineParseGrokProcessorType,), } attribute_map = { "disable_library_rules": "disable_library_rules", + "enabled": "enabled", "id": "id", "include": "include", - "inputs": "inputs", "rules": "rules", "type": "type", } def __init__( self_, + enabled: bool, id: str, include: str, - inputs: List[str], rules: List[ObservabilityPipelineParseGrokProcessorRule], type: ObservabilityPipelineParseGrokProcessorType, disable_library_rules: Union[bool, UnsetType] = unset, @@ -66,15 +66,15 @@ def __init__( :param disable_library_rules: If set to ``true`` , disables the default Grok rules provided by Datadog. :type disable_library_rules: bool, optional + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param id: A unique identifier for this processor. :type id: str :param include: A Datadog search query used to determine which logs this processor targets. :type include: str - :param inputs: A list of component IDs whose output is used as the ``input`` for this component. - :type inputs: [str] - :param rules: The list of Grok parsing rules. If multiple matching rules are provided, they are evaluated in order. The first successful match is applied. :type rules: [ObservabilityPipelineParseGrokProcessorRule] @@ -85,8 +85,8 @@ def __init__( kwargs["disable_library_rules"] = disable_library_rules super().__init__(kwargs) + self_.enabled = enabled self_.id = id self_.include = include - self_.inputs = inputs self_.rules = rules self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_parse_json_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_parse_json_processor.py index ec3357b625..7325c2f261 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_parse_json_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_parse_json_processor.py @@ -3,7 +3,7 @@ # Copyright 2019-Present Datadog, Inc. from __future__ import annotations -from typing import List, TYPE_CHECKING +from typing import TYPE_CHECKING from datadog_api_client.model_utils import ( ModelNormal, @@ -25,33 +25,36 @@ def openapi_types(_): ) return { + "enabled": (bool,), "field": (str,), "id": (str,), "include": (str,), - "inputs": ([str],), "type": (ObservabilityPipelineParseJSONProcessorType,), } attribute_map = { + "enabled": "enabled", "field": "field", "id": "id", "include": "include", - "inputs": "inputs", "type": "type", } def __init__( self_, + enabled: bool, field: str, id: str, include: str, - inputs: List[str], type: ObservabilityPipelineParseJSONProcessorType, **kwargs, ): """ The ``parse_json`` processor extracts JSON from a specified field and flattens it into the event. This is useful when logs contain embedded JSON as a string. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param field: The name of the log field that contains a JSON string. :type field: str @@ -61,16 +64,13 @@ def __init__( :param include: A Datadog search query used to determine which logs this processor targets. :type include: str - :param inputs: A list of component IDs whose output is used as the ``input`` for this component. - :type inputs: [str] - :param type: The processor type. The value should always be ``parse_json``. :type type: ObservabilityPipelineParseJSONProcessorType """ super().__init__(kwargs) + self_.enabled = enabled self_.field = field self_.id = id self_.include = include - self_.inputs = inputs self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_quota_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_quota_processor.py index aa4be5a8ca..e4801029c9 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_quota_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_quota_processor.py @@ -46,10 +46,10 @@ def openapi_types(_): return { "drop_events": (bool,), + "enabled": (bool,), "id": (str,), "ignore_when_missing_partitions": (bool,), "include": (str,), - "inputs": ([str],), "limit": (ObservabilityPipelineQuotaProcessorLimit,), "name": (str,), "overflow_action": (ObservabilityPipelineQuotaProcessorOverflowAction,), @@ -60,10 +60,10 @@ def openapi_types(_): attribute_map = { "drop_events": "drop_events", + "enabled": "enabled", "id": "id", "ignore_when_missing_partitions": "ignore_when_missing_partitions", "include": "include", - "inputs": "inputs", "limit": "limit", "name": "name", "overflow_action": "overflow_action", @@ -74,13 +74,13 @@ def openapi_types(_): def __init__( self_, - drop_events: bool, + enabled: bool, id: str, include: str, - inputs: List[str], limit: ObservabilityPipelineQuotaProcessorLimit, name: str, type: ObservabilityPipelineQuotaProcessorType, + drop_events: Union[bool, UnsetType] = unset, ignore_when_missing_partitions: Union[bool, UnsetType] = unset, overflow_action: Union[ObservabilityPipelineQuotaProcessorOverflowAction, UnsetType] = unset, overrides: Union[List[ObservabilityPipelineQuotaProcessorOverride], UnsetType] = unset, @@ -91,7 +91,10 @@ def __init__( The Quota Processor measures logging traffic for logs that match a specified filter. When the configured daily quota is met, the processor can drop or alert. :param drop_events: If set to ``true`` , logs that matched the quota filter and sent after the quota has been met are dropped; only logs that did not match the filter query continue through the pipeline. - :type drop_events: bool + :type drop_events: bool, optional + + :param enabled: Whether this processor is enabled. + :type enabled: bool :param id: The unique identifier for this component. Used to reference this component in other parts of the pipeline (for example, as the ``input`` to downstream components). :type id: str @@ -102,9 +105,6 @@ def __init__( :param include: A Datadog search query used to determine which logs this processor targets. :type include: str - :param inputs: A list of component IDs whose output is used as the ``input`` for this component. - :type inputs: [str] - :param limit: The maximum amount of data or number of events allowed before the quota is enforced. Can be specified in bytes or events. :type limit: ObservabilityPipelineQuotaProcessorLimit @@ -127,6 +127,8 @@ def __init__( :param type: The processor type. The value should always be ``quota``. :type type: ObservabilityPipelineQuotaProcessorType """ + if drop_events is not unset: + kwargs["drop_events"] = drop_events if ignore_when_missing_partitions is not unset: kwargs["ignore_when_missing_partitions"] = ignore_when_missing_partitions if overflow_action is not unset: @@ -137,10 +139,9 @@ def __init__( kwargs["partition_fields"] = partition_fields super().__init__(kwargs) - self_.drop_events = drop_events + self_.enabled = enabled self_.id = id self_.include = include - self_.inputs = inputs self_.limit = limit self_.name = name self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_reduce_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_reduce_processor.py index bcddc2844d..210ebd9f3d 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_reduce_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_reduce_processor.py @@ -31,29 +31,29 @@ def openapi_types(_): ) return { + "enabled": (bool,), "group_by": ([str],), "id": (str,), "include": (str,), - "inputs": ([str],), "merge_strategies": ([ObservabilityPipelineReduceProcessorMergeStrategy],), "type": (ObservabilityPipelineReduceProcessorType,), } attribute_map = { + "enabled": "enabled", "group_by": "group_by", "id": "id", "include": "include", - "inputs": "inputs", "merge_strategies": "merge_strategies", "type": "type", } def __init__( self_, + enabled: bool, group_by: List[str], id: str, include: str, - inputs: List[str], merge_strategies: List[ObservabilityPipelineReduceProcessorMergeStrategy], type: ObservabilityPipelineReduceProcessorType, **kwargs, @@ -61,6 +61,9 @@ def __init__( """ The ``reduce`` processor aggregates and merges logs based on matching keys and merge strategies. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param group_by: A list of fields used to group log events for merging. :type group_by: [str] @@ -70,9 +73,6 @@ def __init__( :param include: A Datadog search query used to determine which logs this processor targets. :type include: str - :param inputs: A list of component IDs whose output is used as the input for this processor. - :type inputs: [str] - :param merge_strategies: List of merge strategies defining how values from grouped events should be combined. :type merge_strategies: [ObservabilityPipelineReduceProcessorMergeStrategy] @@ -81,9 +81,9 @@ def __init__( """ super().__init__(kwargs) + self_.enabled = enabled self_.group_by = group_by self_.id = id self_.include = include - self_.inputs = inputs self_.merge_strategies = merge_strategies self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_remove_fields_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_remove_fields_processor.py index c8a0d89bc0..ace4d0d308 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_remove_fields_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_remove_fields_processor.py @@ -25,33 +25,36 @@ def openapi_types(_): ) return { + "enabled": (bool,), "fields": ([str],), "id": (str,), "include": (str,), - "inputs": ([str],), "type": (ObservabilityPipelineRemoveFieldsProcessorType,), } attribute_map = { + "enabled": "enabled", "fields": "fields", "id": "id", "include": "include", - "inputs": "inputs", "type": "type", } def __init__( self_, + enabled: bool, fields: List[str], id: str, include: str, - inputs: List[str], type: ObservabilityPipelineRemoveFieldsProcessorType, **kwargs, ): """ The ``remove_fields`` processor deletes specified fields from logs. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param fields: A list of field names to be removed from each log event. :type fields: [str] @@ -61,16 +64,13 @@ def __init__( :param include: A Datadog search query used to determine which logs this processor targets. :type include: str - :param inputs: The ``PipelineRemoveFieldsProcessor`` ``inputs``. - :type inputs: [str] - :param type: The processor type. The value should always be ``remove_fields``. :type type: ObservabilityPipelineRemoveFieldsProcessorType """ super().__init__(kwargs) + self_.enabled = enabled self_.fields = fields self_.id = id self_.include = include - self_.inputs = inputs self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_rename_fields_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_rename_fields_processor.py index ae737aceb1..a9c84cf492 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_rename_fields_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_rename_fields_processor.py @@ -31,33 +31,36 @@ def openapi_types(_): ) return { + "enabled": (bool,), "fields": ([ObservabilityPipelineRenameFieldsProcessorField],), "id": (str,), "include": (str,), - "inputs": ([str],), "type": (ObservabilityPipelineRenameFieldsProcessorType,), } attribute_map = { + "enabled": "enabled", "fields": "fields", "id": "id", "include": "include", - "inputs": "inputs", "type": "type", } def __init__( self_, + enabled: bool, fields: List[ObservabilityPipelineRenameFieldsProcessorField], id: str, include: str, - inputs: List[str], type: ObservabilityPipelineRenameFieldsProcessorType, **kwargs, ): """ The ``rename_fields`` processor changes field names. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param fields: A list of rename rules specifying which fields to rename in the event, what to rename them to, and whether to preserve the original fields. :type fields: [ObservabilityPipelineRenameFieldsProcessorField] @@ -67,16 +70,13 @@ def __init__( :param include: A Datadog search query used to determine which logs this processor targets. :type include: str - :param inputs: A list of component IDs whose output is used as the ``input`` for this component. - :type inputs: [str] - :param type: The processor type. The value should always be ``rename_fields``. :type type: ObservabilityPipelineRenameFieldsProcessorType """ super().__init__(kwargs) + self_.enabled = enabled self_.fields = fields self_.id = id self_.include = include - self_.inputs = inputs self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_sample_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_sample_processor.py index e6f16ce99d..5d4a3bd479 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_sample_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_sample_processor.py @@ -3,7 +3,7 @@ # Copyright 2019-Present Datadog, Inc. from __future__ import annotations -from typing import List, Union, TYPE_CHECKING +from typing import Union, TYPE_CHECKING from datadog_api_client.model_utils import ( ModelNormal, @@ -33,18 +33,18 @@ def openapi_types(_): ) return { + "enabled": (bool,), "id": (str,), "include": (str,), - "inputs": ([str],), "percentage": (float,), "rate": (int,), "type": (ObservabilityPipelineSampleProcessorType,), } attribute_map = { + "enabled": "enabled", "id": "id", "include": "include", - "inputs": "inputs", "percentage": "percentage", "rate": "rate", "type": "type", @@ -52,9 +52,9 @@ def openapi_types(_): def __init__( self_, + enabled: bool, id: str, include: str, - inputs: List[str], type: ObservabilityPipelineSampleProcessorType, percentage: Union[float, UnsetType] = unset, rate: Union[int, UnsetType] = unset, @@ -63,15 +63,15 @@ def __init__( """ The ``sample`` processor allows probabilistic sampling of logs at a fixed rate. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param id: The unique identifier for this component. Used to reference this component in other parts of the pipeline (for example, as the ``input`` to downstream components). :type id: str :param include: A Datadog search query used to determine which logs this processor targets. :type include: str - :param inputs: A list of component IDs whose output is used as the ``input`` for this component. - :type inputs: [str] - :param percentage: The percentage of logs to sample. :type percentage: float, optional @@ -87,7 +87,7 @@ def __init__( kwargs["rate"] = rate super().__init__(kwargs) + self_.enabled = enabled self_.id = id self_.include = include - self_.inputs = inputs self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_sensitive_data_scanner_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_sensitive_data_scanner_processor.py index 9d78e97932..a6a755aa76 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_sensitive_data_scanner_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_sensitive_data_scanner_processor.py @@ -31,26 +31,26 @@ def openapi_types(_): ) return { + "enabled": (bool,), "id": (str,), "include": (str,), - "inputs": ([str],), "rules": ([ObservabilityPipelineSensitiveDataScannerProcessorRule],), "type": (ObservabilityPipelineSensitiveDataScannerProcessorType,), } attribute_map = { + "enabled": "enabled", "id": "id", "include": "include", - "inputs": "inputs", "rules": "rules", "type": "type", } def __init__( self_, + enabled: bool, id: str, include: str, - inputs: List[str], rules: List[ObservabilityPipelineSensitiveDataScannerProcessorRule], type: ObservabilityPipelineSensitiveDataScannerProcessorType, **kwargs, @@ -58,15 +58,15 @@ def __init__( """ The ``sensitive_data_scanner`` processor detects and optionally redacts sensitive data in log events. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param id: The unique identifier for this component. Used to reference this component in other parts of the pipeline (e.g., as input to downstream components). :type id: str :param include: A Datadog search query used to determine which logs this processor targets. :type include: str - :param inputs: A list of component IDs whose output is used as the ``input`` for this component. - :type inputs: [str] - :param rules: A list of rules for identifying and acting on sensitive data patterns. :type rules: [ObservabilityPipelineSensitiveDataScannerProcessorRule] @@ -75,8 +75,8 @@ def __init__( """ super().__init__(kwargs) + self_.enabled = enabled self_.id = id self_.include = include - self_.inputs = inputs self_.rules = rules self_.type = type diff --git a/src/datadog_api_client/v2/model/observability_pipeline_throttle_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_throttle_processor.py index aeaa56e6cb..14c7c559e7 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_throttle_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_throttle_processor.py @@ -27,20 +27,20 @@ def openapi_types(_): ) return { + "enabled": (bool,), "group_by": ([str],), "id": (str,), "include": (str,), - "inputs": ([str],), "threshold": (int,), "type": (ObservabilityPipelineThrottleProcessorType,), "window": (float,), } attribute_map = { + "enabled": "enabled", "group_by": "group_by", "id": "id", "include": "include", - "inputs": "inputs", "threshold": "threshold", "type": "type", "window": "window", @@ -48,9 +48,9 @@ def openapi_types(_): def __init__( self_, + enabled: bool, id: str, include: str, - inputs: List[str], threshold: int, type: ObservabilityPipelineThrottleProcessorType, window: float, @@ -60,6 +60,9 @@ def __init__( """ The ``throttle`` processor limits the number of events that pass through over a given time window. + :param enabled: Whether this processor is enabled. + :type enabled: bool + :param group_by: Optional list of fields used to group events before the threshold has been reached. :type group_by: [str], optional @@ -69,9 +72,6 @@ def __init__( :param include: A Datadog search query used to determine which logs this processor targets. :type include: str - :param inputs: A list of component IDs whose output is used as the input for this processor. - :type inputs: [str] - :param threshold: the number of events allowed in a given time window. Events sent after the threshold has been reached, are dropped. :type threshold: int @@ -85,9 +85,9 @@ def __init__( kwargs["group_by"] = group_by super().__init__(kwargs) + self_.enabled = enabled self_.id = id self_.include = include - self_.inputs = inputs self_.threshold = threshold self_.type = type self_.window = window diff --git a/src/datadog_api_client/v2/models/__init__.py b/src/datadog_api_client/v2/models/__init__.py index 9cece11cff..6f2757bec1 100644 --- a/src/datadog_api_client/v2/models/__init__.py +++ b/src/datadog_api_client/v2/models/__init__.py @@ -2820,6 +2820,9 @@ from datadog_api_client.v2.model.observability_pipeline_config_destination_item import ( ObservabilityPipelineConfigDestinationItem, ) +from datadog_api_client.v2.model.observability_pipeline_config_processor_group import ( + ObservabilityPipelineConfigProcessorGroup, +) from datadog_api_client.v2.model.observability_pipeline_config_processor_item import ( ObservabilityPipelineConfigProcessorItem, ) @@ -7068,6 +7071,7 @@ "ObservabilityPipelineAwsAuth", "ObservabilityPipelineConfig", "ObservabilityPipelineConfigDestinationItem", + "ObservabilityPipelineConfigProcessorGroup", "ObservabilityPipelineConfigProcessorItem", "ObservabilityPipelineConfigSourceItem", "ObservabilityPipelineCrowdStrikeNextGenSiemDestination", diff --git a/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_bad_request_response.frozen b/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_bad_request_response.frozen index 71a6a09485..c9a3812ad6 100644 --- a/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_bad_request_response.frozen +++ b/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_bad_request_response.frozen @@ -1 +1 @@ -2025-04-25T17:29:21.625Z \ No newline at end of file +2025-11-26T16:03:26.936Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_bad_request_response.yaml b/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_bad_request_response.yaml index fd22432688..0099df74f0 100644 --- a/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_bad_request_response.yaml +++ b/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_bad_request_response.yaml @@ -1,6 +1,6 @@ interactions: - request: - body: '{"data":{"attributes":{"config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"unknown-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Main + body: '{"data":{"attributes":{"config":{"destinations":[{"id":"datadog-logs-destination","inputs":["my-processor-group"],"type":"datadog_logs"}],"processors":[{"enabled":true,"id":"unknown-processor","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"enabled":true,"id":"filter-processor","include":"status:error","type":"filter"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Main Observability Pipeline"},"type":"pipelines"}}' headers: accept: @@ -11,8 +11,8 @@ interactions: uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines response: body: - string: '{"errors":[{"title":"Component with ID filter-processor is an unknown - component","meta":{"message":"Component with ID filter-processor is an unknown + string: '{"errors":[{"title":"Component with ID my-processor-group is an unknown + component","meta":{"message":"Component with ID my-processor-group is an unknown component"}},{"title":"The following components are unused: [datadog-agent-source unknown-processor]","meta":{"message":"The following components are unused: [datadog-agent-source unknown-processor]"}}]} diff --git a/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_ok_response.frozen b/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_ok_response.frozen index a6261e736f..c95d315fed 100644 --- a/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_ok_response.frozen +++ b/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_ok_response.frozen @@ -1 +1 @@ -2025-04-25T17:29:23.837Z \ No newline at end of file +2025-11-26T16:03:27.483Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_ok_response.yaml b/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_ok_response.yaml index 165507fcc0..d912d74f10 100644 --- a/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_ok_response.yaml +++ b/tests/v2/cassettes/test_scenarios/test_create_a_new_pipeline_returns_ok_response.yaml @@ -1,6 +1,6 @@ interactions: - request: - body: '{"data":{"attributes":{"config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Main + body: '{"data":{"attributes":{"config":{"destinations":[{"id":"datadog-logs-destination","inputs":["my-processor-group"],"type":"datadog_logs"}],"processors":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"enabled":true,"id":"filter-processor","include":"status:error","type":"filter"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Main Observability Pipeline"},"type":"pipelines"}}' headers: accept: @@ -11,8 +11,8 @@ interactions: uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines response: body: - string: '{"data":{"id":"d891e45c-21fa-11f0-96dc-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}}} + string: '{"data":{"id":"7231fee6-cae1-11f0-a2c3-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["my-processor-group"],"type":"datadog_logs"}],"processors":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"enabled":true,"id":"filter-processor","include":"status:error","type":"filter"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}}} ' headers: @@ -27,7 +27,7 @@ interactions: accept: - '*/*' method: DELETE - uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/d891e45c-21fa-11f0-96dc-da7ad0900002 + uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/7231fee6-cae1-11f0-a2c3-da7ad0900002 response: body: string: '' diff --git a/tests/v2/cassettes/test_scenarios/test_delete_a_pipeline_returns_not_found_response.frozen b/tests/v2/cassettes/test_scenarios/test_delete_a_pipeline_returns_not_found_response.frozen index 033a0755d0..0c75345fb8 100644 --- a/tests/v2/cassettes/test_scenarios/test_delete_a_pipeline_returns_not_found_response.frozen +++ b/tests/v2/cassettes/test_scenarios/test_delete_a_pipeline_returns_not_found_response.frozen @@ -1 +1 @@ -2025-04-25T17:29:25.292Z \ No newline at end of file +2025-11-26T16:03:29.108Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_delete_a_pipeline_returns_ok_response.frozen b/tests/v2/cassettes/test_scenarios/test_delete_a_pipeline_returns_ok_response.frozen index 9cbc728e00..48d506876c 100644 --- a/tests/v2/cassettes/test_scenarios/test_delete_a_pipeline_returns_ok_response.frozen +++ b/tests/v2/cassettes/test_scenarios/test_delete_a_pipeline_returns_ok_response.frozen @@ -1 +1 @@ -2025-04-25T17:29:26.847Z \ No newline at end of file +2025-11-26T16:03:29.684Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_delete_a_pipeline_returns_ok_response.yaml b/tests/v2/cassettes/test_scenarios/test_delete_a_pipeline_returns_ok_response.yaml index a06d7942c9..9a32f53752 100644 --- a/tests/v2/cassettes/test_scenarios/test_delete_a_pipeline_returns_ok_response.yaml +++ b/tests/v2/cassettes/test_scenarios/test_delete_a_pipeline_returns_ok_response.yaml @@ -11,7 +11,7 @@ interactions: uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines response: body: - string: '{"data":{"id":"da5f2218-21fa-11f0-96de-da7ad0900002","type":"pipelines","attributes":{"name":"Main + string: '{"data":{"id":"73822ece-cae1-11f0-a2c5-da7ad0900002","type":"pipelines","attributes":{"name":"Main Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}}} ' @@ -27,7 +27,7 @@ interactions: accept: - '*/*' method: DELETE - uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/da5f2218-21fa-11f0-96de-da7ad0900002 + uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/73822ece-cae1-11f0-a2c5-da7ad0900002 response: body: string: '' @@ -43,7 +43,7 @@ interactions: accept: - '*/*' method: DELETE - uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/da5f2218-21fa-11f0-96de-da7ad0900002 + uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/73822ece-cae1-11f0-a2c5-da7ad0900002 response: body: string: '{"errors":[{"title":"Resource Not Found"}]} diff --git a/tests/v2/cassettes/test_scenarios/test_get_a_specific_pipeline_returns_ok_response.frozen b/tests/v2/cassettes/test_scenarios/test_get_a_specific_pipeline_returns_ok_response.frozen index f863fd1ac2..55c24212d8 100644 --- a/tests/v2/cassettes/test_scenarios/test_get_a_specific_pipeline_returns_ok_response.frozen +++ b/tests/v2/cassettes/test_scenarios/test_get_a_specific_pipeline_returns_ok_response.frozen @@ -1 +1 @@ -2025-04-25T17:29:29.196Z \ No newline at end of file +2025-11-26T16:03:32.309Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_get_a_specific_pipeline_returns_ok_response.yaml b/tests/v2/cassettes/test_scenarios/test_get_a_specific_pipeline_returns_ok_response.yaml index c680773358..2568ceb811 100644 --- a/tests/v2/cassettes/test_scenarios/test_get_a_specific_pipeline_returns_ok_response.yaml +++ b/tests/v2/cassettes/test_scenarios/test_get_a_specific_pipeline_returns_ok_response.yaml @@ -11,7 +11,7 @@ interactions: uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines response: body: - string: '{"data":{"id":"dbbff3d0-21fa-11f0-96e0-da7ad0900002","type":"pipelines","attributes":{"name":"Main + string: '{"data":{"id":"750ea1f0-cae1-11f0-a2c7-da7ad0900002","type":"pipelines","attributes":{"name":"Main Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}}} ' @@ -27,10 +27,10 @@ interactions: accept: - application/json method: GET - uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/dbbff3d0-21fa-11f0-96e0-da7ad0900002 + uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/750ea1f0-cae1-11f0-a2c7-da7ad0900002 response: body: - string: '{"data":{"id":"dbbff3d0-21fa-11f0-96e0-da7ad0900002","type":"pipelines","attributes":{"name":"Main + string: '{"data":{"id":"750ea1f0-cae1-11f0-a2c7-da7ad0900002","type":"pipelines","attributes":{"name":"Main Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}}} ' @@ -46,7 +46,7 @@ interactions: accept: - '*/*' method: DELETE - uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/dbbff3d0-21fa-11f0-96e0-da7ad0900002 + uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/750ea1f0-cae1-11f0-a2c7-da7ad0900002 response: body: string: '' diff --git a/tests/v2/cassettes/test_scenarios/test_list_pipelines_returns_bad_request_response.frozen b/tests/v2/cassettes/test_scenarios/test_list_pipelines_returns_bad_request_response.frozen index 3a7978da8a..e56bb9cfbd 100644 --- a/tests/v2/cassettes/test_scenarios/test_list_pipelines_returns_bad_request_response.frozen +++ b/tests/v2/cassettes/test_scenarios/test_list_pipelines_returns_bad_request_response.frozen @@ -1 +1 @@ -2025-04-25T17:29:31.769Z \ No newline at end of file +2025-11-26T16:03:35.391Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_list_pipelines_returns_ok_response.frozen b/tests/v2/cassettes/test_scenarios/test_list_pipelines_returns_ok_response.frozen index 4c9f3cfcbc..a879a77baf 100644 --- a/tests/v2/cassettes/test_scenarios/test_list_pipelines_returns_ok_response.frozen +++ b/tests/v2/cassettes/test_scenarios/test_list_pipelines_returns_ok_response.frozen @@ -1 +1 @@ -2025-04-25T17:29:32.322Z \ No newline at end of file +2025-11-26T16:03:35.904Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_list_pipelines_returns_ok_response.yaml b/tests/v2/cassettes/test_scenarios/test_list_pipelines_returns_ok_response.yaml index 3c0f76dafe..1447917a0a 100644 --- a/tests/v2/cassettes/test_scenarios/test_list_pipelines_returns_ok_response.yaml +++ b/tests/v2/cassettes/test_scenarios/test_list_pipelines_returns_ok_response.yaml @@ -11,7 +11,7 @@ interactions: uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines response: body: - string: '{"data":{"id":"dd87c652-21fa-11f0-96e2-da7ad0900002","type":"pipelines","attributes":{"name":"Main + string: '{"data":{"id":"77329f9a-cae1-11f0-a2c9-da7ad0900002","type":"pipelines","attributes":{"name":"Main Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}}} ' @@ -30,32 +30,42 @@ interactions: uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines response: body: - string: '{"data":[{"id":"8d85d864-0f09-11f0-9711-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"3f339054-10ab-11f0-88a2-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"b8c068de-10ab-11f0-88a8-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"32e2f90a-1139-11f0-8501-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"67e79020-1139-11f0-98d1-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"30966e06-113a-11f0-98e1-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"3e3f9382-138f-11f0-8cf0-da7ad0900002","type":"pipelines","attributes":{"name":"test - pipeline","config":{"destinations":[{"id":"destination-1","inputs":["parser-1"],"type":"datadog_logs"}],"processors":[{"field":"message","id":"parser-1","include":"service:my-service","inputs":["source-1"],"type":"parse_json"}],"sources":[{"id":"source-1","tls":{"crt_file":"/path/to/cert.crt"},"type":"datadog_agent"}]}}},{"id":"42159650-138f-11f0-a2aa-da7ad0900002","type":"pipelines","attributes":{"name":"test - pipeline","config":{"destinations":[{"id":"destination-1","inputs":["parser-1"],"type":"datadog_logs"}],"processors":[{"field":"message","id":"parser-1","include":"service:my-service","inputs":["source-1"],"type":"parse_json"}],"sources":[{"id":"source-1","tls":{"crt_file":"/path/to/cert.crt"},"type":"datadog_agent"}]}}},{"id":"6cc001f8-1392-11f0-9e35-da7ad0900002","type":"pipelines","attributes":{"name":"agent - with tls","config":{"destinations":[{"id":"destination-1","inputs":["source-with-tls"],"type":"datadog_logs"}],"processors":[],"sources":[{"id":"source-with-tls","tls":{"ca_file":"/etc/certs/ca.crt","crt_file":"/etc/certs/agent.crt","key_file":"/etc/certs/agent.key"},"type":"datadog_agent"}]}}},{"id":"bdf5078e-139d-11f0-8e4f-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"e67402d2-139d-11f0-8e51-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"e9849f22-139d-11f0-8e53-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"f9845bb0-139d-11f0-b101-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"2ebd82ca-139e-11f0-b103-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"31ba5a34-139e-11f0-8e55-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"3aa802c2-139e-11f0-8e57-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"7b3daaf8-139e-11f0-8e59-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"7e72d32e-139e-11f0-8e5b-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"818a8728-139e-11f0-8e5d-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"bed86e88-139e-11f0-8e5f-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"c202b050-139e-11f0-b107-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"c5111084-139e-11f0-8e61-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"e01ae6de-139e-11f0-8e63-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"e33ad356-139e-11f0-b109-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"e6783e78-139e-11f0-b10b-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"04407bfa-139f-11f0-8e65-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"126be958-139f-11f0-8e67-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"97607354-139f-11f0-8e6b-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"305f9a88-13a2-11f0-8dd9-da7ad0900002","type":"pipelines","attributes":{"name":"parse-json-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["parser-1"],"type":"datadog_logs"}],"processors":[{"field":"message","id":"parser-1","include":"env:parse","inputs":["source-1"],"type":"parse_json"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"45669120-13a2-11f0-8ddb-da7ad0900002","type":"pipelines","attributes":{"name":"parse-json-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["parser-1"],"type":"datadog_logs"}],"processors":[{"field":"message","id":"parser-1","include":"env:parse","inputs":["source-1"],"type":"parse_json"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"c00f7548-13a4-11f0-9315-da7ad0900002","type":"pipelines","attributes":{"name":"test - pipeline","config":{"destinations":[{"id":"destination-1","inputs":["parser-1"],"type":"datadog_logs"}],"processors":[{"field":"message","id":"parser-1","include":"service:my-service","inputs":["source-1"],"type":"parse_json"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"c07cf206-13a6-11f0-949d-da7ad0900002","type":"pipelines","attributes":{"name":"test - pipeline","config":{"destinations":[{"id":"destination-1","inputs":["parser-1"],"type":"datadog_logs"}],"processors":[{"field":"message","id":"parser-1","include":"service:my-service","inputs":["source-1"],"type":"parse_json"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"8ff06e36-13ad-11f0-8243-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"8ff1a508-13ad-11f0-b934-da7ad0900002","type":"pipelines","attributes":{"name":"agent - with tls","config":{"destinations":[{"id":"destination-1","inputs":["source-with-tls"],"type":"datadog_logs"}],"processors":[],"sources":[{"id":"source-with-tls","tls":{"crt_file":"/etc/certs/agent.crt","key_file":"/etc/certs/agent.key"},"type":"datadog_agent"}]}}},{"id":"8ff38a08-13ad-11f0-8245-da7ad0900002","type":"pipelines","attributes":{"name":"parse-json-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["parser-1"],"type":"datadog_logs"}],"processors":[{"field":"message","id":"parser-1","include":"env:parse","inputs":["source-1"],"type":"parse_json"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"8ff54f0a-13ad-11f0-8247-da7ad0900002","type":"pipelines","attributes":{"name":"kafka - pipeline","config":{"destinations":[{"id":"destination-1","inputs":["kafka-source-1"],"type":"datadog_logs"}],"processors":[],"sources":[{"group_id":"consumer-group-1","id":"kafka-source-1","sasl":{"mechanism":"PLAIN"},"tls":{"ca_file":"","crt_file":"/path/to/kafka.crt"},"topics":["topic-a","topic-b"],"type":"kafka"}]}}},{"id":"f814235e-13ad-11f0-b942-da7ad0900002","type":"pipelines","attributes":{"name":"agent - with tls","config":{"destinations":[{"id":"destination-1","inputs":["source-with-tls"],"type":"datadog_logs"}],"processors":[],"sources":[{"id":"source-with-tls","tls":{"crt_file":"/etc/certs/agent.crt","key_file":"/etc/certs/agent.key"},"type":"datadog_agent"}]}}},{"id":"5b1a9314-13ce-11f0-a9de-da7ad0900002","type":"pipelines","attributes":{"name":"rename-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["rename-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"destination":"new.field","preserve_source":true,"source":"old.field"}],"id":"rename-1","include":"*","inputs":["source-1"],"type":"rename_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"1e356802-1419-11f0-8834-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"14156a86-142b-11f0-96f3-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"a5280a06-143a-11f0-aac4-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"5005f6c2-1481-11f0-8faa-da7ad0900002","type":"pipelines","attributes":{"name":"remove-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["remove-1"],"type":"datadog_logs"}],"processors":[{"fields":["temp.debug","internal.trace_id"],"id":"remove-1","include":"*","inputs":["source-1"],"type":"remove_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"9c1776c0-14d7-11f0-87c9-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"b1e4c58c-1501-11f0-b0bd-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"43a8a4cc-15c3-11f0-b111-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"45042a58-15c3-11f0-b113-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"f9de3956-15cc-11f0-ac43-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"b3efaa52-1693-11f0-89e0-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"24c83620-1696-11f0-89e6-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"5d2875d6-17a2-11f0-9bd1-da7ad0900002","type":"pipelines","attributes":{"name":"Main - Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}}],"meta":{"totalCount":128}} + string: '{"data":[{"id":"79e292e4-7eed-11f0-ae9a-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"95f5859a-7eed-11f0-b11e-da7ad0900002","type":"pipelines","attributes":{"name":"remove-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["remove-1"],"type":"datadog_logs"}],"processors":[{"fields":["temp.debug","internal.trace_id"],"id":"remove-1","include":"*","inputs":["source-1"],"type":"remove_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"befe717c-7eed-11f0-b142-da7ad0900002","type":"pipelines","attributes":{"name":"sentinel-pipeline","config":{"destinations":[{"client_id":"a1b2c3d4-5678-90ab-cdef-1234567890ab","dcr_immutable_id":"dcr-uuid-1234","id":"sentinel-dest-1","inputs":["source-1"],"table":"CustomLogsTable","tenant_id":"abcdef12-3456-7890-abcd-ef1234567890","type":"microsoft_sentinel"}],"processors":[],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"cde707bc-7eed-11f0-b14e-da7ad0900002","type":"pipelines","attributes":{"name":"reduce + pipeline","config":{"destinations":[{"id":"destination-1","inputs":["reduce-1"],"type":"datadog_logs"}],"processors":[{"group_by":["log.user.id","log.device.id"],"id":"reduce-1","include":"env:prod","inputs":["source-1"],"merge_strategies":[{"path":"log.user.roles","strategy":"flat_unique"},{"path":"log.error.messages","strategy":"concat"},{"path":"log.count","strategy":"sum"},{"path":"log.status","strategy":"retain"}],"type":"reduce"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"d48b3778-7eed-11f0-b156-da7ad0900002","type":"pipelines","attributes":{"name":"chronicle + pipeline","config":{"destinations":[{"auth":{"credentials_file":"/secrets/gcp.json"},"customer_id":"3fa85f64-5717-4562-b3fc-2c963f66afa6","encoding":"json","id":"chronicle-dest-1","inputs":["source-1"],"log_type":"nginx_logs","type":"google_chronicle"}],"processors":[],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"a7b4ba96-7f09-11f0-b21f-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"02f9b5d4-7f17-11f0-b042-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"92b580a6-7f4c-11f0-b432-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"1dbda0ee-7f51-11f0-9018-da7ad0900002","type":"pipelines","attributes":{"name":"firehose + pipeline","config":{"destinations":[{"id":"destination-1","inputs":["firehose-source-1"],"type":"datadog_logs"}],"processors":[],"sources":[{"auth":{"assume_role":"arn:aws:iam::123456789012:role/ExampleRole","external_id":"external-id-123","session_name":"firehose-session"},"id":"firehose-source-1","tls":{"crt_file":"/path/to/firehose.crt"},"type":"amazon_data_firehose"}]}}},{"id":"23e99d42-7f51-11f0-901c-da7ad0900002","type":"pipelines","attributes":{"name":"dedupe + pipeline","config":{"destinations":[{"id":"destination-1","inputs":["dedupe-ignore"],"type":"datadog_logs"}],"processors":[{"fields":["log.message","log.tags"],"id":"dedupe-match","include":"*","inputs":["source-1"],"mode":"match","type":"dedupe"},{"fields":["log.source","log.context"],"id":"dedupe-ignore","include":"*","inputs":["dedupe-match"],"mode":"ignore","type":"dedupe"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"2686dd3a-7f51-11f0-844e-da7ad0900002","type":"pipelines","attributes":{"name":"throttle + pipeline","config":{"destinations":[{"id":"destination-1","inputs":["throttle-grouped"],"type":"datadog_logs"}],"processors":[{"id":"throttle-global","include":"*","inputs":["source-1"],"threshold":1000,"type":"throttle","window":60},{"group_by":["log.user.id","log.level"],"id":"throttle-grouped","include":"*","inputs":["throttle-global"],"threshold":100,"type":"throttle","window":10}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"342f7956-7f51-11f0-9026-da7ad0900002","type":"pipelines","attributes":{"name":"amazon + opensearch pipeline","config":{"destinations":[{"auth":{"assume_role":"arn:aws:iam::123456789012:role/example-role","aws_region":"us-east-1","external_id":"external-id-123","session_name":"aos-session","strategy":"aws"},"bulk_index":"logs-datastream","id":"aos-dest-1","inputs":["source-1"],"type":"amazon_opensearch"}],"processors":[],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"43159012-7f52-11f0-916f-da7ad0900002","type":"pipelines","attributes":{"name":"test + pipeline","config":{"destinations":[{"id":"destination-1","inputs":["parser-1"],"type":"datadog_logs"}],"processors":[{"field":"message","id":"parser-1","include":"service:my-service","inputs":["source-1"],"type":"parse_json"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"d8b76f78-7fb6-11f0-9f78-da7ad0900002","type":"pipelines","attributes":{"name":"sds-full-test","config":{"destinations":[{"id":"sink","inputs":["sds-1"],"type":"datadog_logs"}],"processors":[{"id":"sds-1","include":"*","inputs":["source-a"],"rules":[{"keyword_options":{"keywords":["secret","token"],"proximity":4},"name":"Redact + with Exclude","on_match":{"action":"redact","options":{"replace":"[REDACTED]"}},"pattern":{"options":{"rule":"\\bsecret-[a-z0-9]+\\b"},"type":"custom"},"scope":{"options":{"fields":["not_this_field"]},"target":"exclude"},"tags":["confidential","mask"]},{"name":"Library + Hash","on_match":{"action":"hash"},"pattern":{"options":{"id":"ip_address","use_recommended_keywords":true},"type":"library"},"scope":{"target":"all"},"tags":["pii"]},{"name":"Partial + Default Scope","on_match":{"action":"partial_redact","options":{"characters":3,"direction":"first"}},"pattern":{"options":{"rule":"user\\d{3,}"},"type":"custom"},"scope":{"options":{"fields":["this_field_only"]},"target":"include"},"tags":["user","pii"]}],"type":"sensitive_data_scanner"}],"sources":[{"id":"source-a","type":"datadog_agent"}]}}},{"id":"e2b4e2ee-7fb6-11f0-9f7e-da7ad0900002","type":"pipelines","attributes":{"name":"dedupe + pipeline","config":{"destinations":[{"id":"destination-1","inputs":["dedupe-ignore"],"type":"datadog_logs"}],"processors":[{"fields":["log.message","log.tags"],"id":"dedupe-match","include":"*","inputs":["source-1"],"mode":"match","type":"dedupe"},{"fields":["log.source","log.context"],"id":"dedupe-ignore","include":"*","inputs":["dedupe-match"],"mode":"ignore","type":"dedupe"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"034843c0-7fb7-11f0-9eb6-da7ad0900002","type":"pipelines","attributes":{"name":"remap-vrl-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["remap-processor-1"],"type":"datadog_logs"}],"processors":[{"id":"remap-processor-1","include":"*","inputs":["source-1"],"remaps":[{"drop_on_error":false,"enabled":true,"include":"service:web","name":"Parse + JSON from message","source":". = parse_json!(string!(.message))"},{"drop_on_error":true,"enabled":true,"include":"env:prod","name":"Add + timestamp","source":".timestamp = now()"}],"type":"custom_processor"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"3ccd875c-7fd2-11f0-b6d7-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"6b30d2f0-7fd9-11f0-af48-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"fcf10236-7fdf-11f0-8692-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"a58467e6-8015-11f0-9924-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"bb15a8a4-8015-11f0-a7c1-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"05889c3a-801a-11f0-a7d5-da7ad0900002","type":"pipelines","attributes":{"name":"sample-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["sample-2"],"type":"datadog_logs"}],"processors":[{"id":"sample-1","include":"*","inputs":["source-1"],"rate":10,"type":"sample"},{"id":"sample-2","include":"*","inputs":["sample-1"],"percentage":4.99,"type":"sample"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"0a28d246-801a-11f0-8394-da7ad0900002","type":"pipelines","attributes":{"name":"http-server-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["http-source-1"],"type":"datadog_logs"}],"processors":[],"sources":[{"auth_strategy":"plain","decoding":"json","id":"http-source-1","tls":{"ca_file":"/etc/ssl/certs/ca.crt","crt_file":"/etc/ssl/certs/http.crt","key_file":"/etc/ssl/private/http.key"},"type":"http_server"}]}}},{"id":"2eb40eaa-801a-11f0-83a6-da7ad0900002","type":"pipelines","attributes":{"name":"logstash + pipeline","config":{"destinations":[{"id":"destination-1","inputs":["logstash-source-1"],"type":"datadog_logs"}],"processors":[],"sources":[{"id":"logstash-source-1","tls":{"crt_file":"/path/to/logstash.crt"},"type":"logstash"}]}}},{"id":"4c6ad8b6-801a-11f0-83b4-da7ad0900002","type":"pipelines","attributes":{"name":"socket-pipeline-udp","config":{"destinations":[{"id":"destination-1","inputs":["socket-source-2"],"type":"datadog_logs"}],"processors":[],"sources":[{"framing":{"delimiter":"|","method":"character_delimited"},"id":"socket-source-2","mode":"udp","type":"socket"}]}}},{"id":"4f07b526-801a-11f0-83b6-da7ad0900002","type":"pipelines","attributes":{"name":"socket-destination-pipeline","config":{"destinations":[{"encoding":"json","framing":{"method":"newline_delimited"},"id":"socket-dest-1","inputs":["source-1"],"mode":"tcp","tls":{"ca_file":"/etc/ssl/certs/ca.crt","crt_file":"/etc/ssl/certs/socket.crt","key_file":"/etc/ssl/private/socket.key"},"type":"socket"}],"processors":[],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"cc14120c-807f-11f0-b4a4-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"42a10830-8080-11f0-b4b6-da7ad0900002","type":"pipelines","attributes":{"name":"http-server-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["http-source-1"],"type":"datadog_logs"}],"processors":[],"sources":[{"auth_strategy":"plain","decoding":"json","id":"http-source-1","tls":{"ca_file":"/etc/ssl/certs/ca.crt","crt_file":"/etc/ssl/certs/http.crt","key_file":"/etc/ssl/private/http.key"},"type":"http_server"}]}}},{"id":"455121dc-8080-11f0-b4b8-da7ad0900002","type":"pipelines","attributes":{"name":"splunk-hec-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["splunk-hec-source-1"],"type":"datadog_logs"}],"processors":[],"sources":[{"id":"splunk-hec-source-1","tls":{"ca_file":"/etc/ssl/certs/ca.crt","crt_file":"/etc/ssl/certs/splunk.crt","key_file":"/etc/ssl/private/splunk.key"},"type":"splunk_hec"}]}}},{"id":"49d9a396-8080-11f0-98a3-da7ad0900002","type":"pipelines","attributes":{"name":"gcs-destination-pipeline","config":{"destinations":[{"acl":"project-private","auth":{"credentials_file":"/var/secrets/gcp-creds.json"},"bucket":"my-gcs-bucket","id":"gcs-destination-1","inputs":["source-1"],"key_prefix":"logs/","metadata":[{"name":"environment","value":"production"},{"name":"team","value":"platform"}],"storage_class":"NEARLINE","type":"google_cloud_storage"}],"processors":[],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"4a9d5098-8080-11f0-b4bc-da7ad0900002","type":"pipelines","attributes":{"name":"splunk-hec-destination-pipeline","config":{"destinations":[{"auto_extract_timestamp":true,"encoding":"json","id":"splunk-hec-1","index":"main","inputs":["source-1"],"sourcetype":"custom_sourcetype","type":"splunk_hec"}],"processors":[],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"4b5b9238-8080-11f0-98a5-da7ad0900002","type":"pipelines","attributes":{"name":"sumo + pipeline","config":{"destinations":[{"encoding":"json","header_custom_fields":[{"name":"X-Sumo-Category","value":"my-app-logs"},{"name":"X-Custom-Header","value":"debug=true"}],"header_host_name":"host-123","header_source_category":"source-category","header_source_name":"source-name","id":"sumo-dest-1","inputs":["source-1"],"type":"sumo_logic"}],"processors":[],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"4fd012a8-8080-11f0-98a9-da7ad0900002","type":"pipelines","attributes":{"name":"rsyslog-dest-pipeline","config":{"destinations":[{"id":"rsyslog-destination-1","inputs":["source-1"],"keepalive":60000,"tls":{"crt_file":"/etc/certs/rsyslog.crt"},"type":"rsyslog"}],"processors":[],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"6d48f746-8080-11f0-98b7-da7ad0900002","type":"pipelines","attributes":{"name":"chronicle + pipeline","config":{"destinations":[{"auth":{"credentials_file":"/secrets/gcp.json"},"customer_id":"3fa85f64-5717-4562-b3fc-2c963f66afa6","encoding":"json","id":"chronicle-dest-1","inputs":["source-1"],"log_type":"nginx_logs","type":"google_chronicle"}],"processors":[],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"73a20eca-8080-11f0-98b9-da7ad0900002","type":"pipelines","attributes":{"name":"opensearch + pipeline","config":{"destinations":[{"bulk_index":"logs-datastream","id":"opensearch-dest-1","inputs":["source-1"],"type":"opensearch"}],"processors":[],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"5e6dae8c-80bd-11f0-a213-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"5f6fbb72-80bd-11f0-a215-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"e7857b20-80de-11f0-a262-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"579ccd4c-80e3-11f0-a296-da7ad0900002","type":"pipelines","attributes":{"name":"syslogng-source-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["syslogng-source-1"],"type":"datadog_logs"}],"processors":[],"sources":[{"id":"syslogng-source-1","mode":"udp","tls":{"crt_file":"/etc/certs/syslogng.crt"},"type":"syslog_ng"}]}}},{"id":"58505402-80e3-11f0-a298-da7ad0900002","type":"pipelines","attributes":{"name":"rsyslog-dest-pipeline","config":{"destinations":[{"id":"rsyslog-destination-1","inputs":["source-1"],"keepalive":60000,"tls":{"crt_file":"/etc/certs/rsyslog.crt"},"type":"rsyslog"}],"processors":[],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"64a3e372-80e3-11f0-a2a6-da7ad0900002","type":"pipelines","attributes":{"name":"firehose + pipeline","config":{"destinations":[{"id":"destination-1","inputs":["firehose-source-1"],"type":"datadog_logs"}],"processors":[],"sources":[{"auth":{"assume_role":"arn:aws:iam::123456789012:role/ExampleRole","external_id":"external-id-123","session_name":"firehose-session"},"id":"firehose-source-1","tls":{"crt_file":"/path/to/firehose.crt"},"type":"amazon_data_firehose"}]}}},{"id":"92751474-80e3-11f0-a2d0-da7ad0900002","type":"pipelines","attributes":{"name":"amazon + s3 pipeline (minimal)","config":{"destinations":[{"bucket":"my-logs-bucket","id":"s3-dest-basic-1","inputs":["source-1"],"key_prefix":"logs/","region":"us-east-1","storage_class":"STANDARD","type":"amazon_s3"}],"processors":[],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"2ae24eb0-8149-11f0-baea-da7ad0900002","type":"pipelines","attributes":{"name":"quota-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["quota-1"],"type":"datadog_logs"}],"processors":[{"drop_events":true,"id":"quota-1","ignore_when_missing_partitions":true,"include":"*","inputs":["source-1"],"limit":{"enforce":"events","limit":1000},"name":"limitByHostAndEnv","overrides":[{"fields":[{"name":"env","value":"prod"},{"name":"host","value":"*"}],"limit":{"enforce":"events","limit":500}},{"fields":[{"name":"env","value":"*"},{"name":"host","value":"localhost"}],"limit":{"enforce":"bytes","limit":300}}],"partition_fields":["host","env"],"type":"quota"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"2d5cca94-8149-11f0-baec-da7ad0900002","type":"pipelines","attributes":{"name":"add-fields-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["add-fields-1"],"type":"datadog_logs"}],"processors":[{"fields":[{"name":"custom.field","value":"hello-world"},{"name":"env","value":"prod"}],"id":"add-fields-1","include":"*","inputs":["source-1"],"type":"add_fields"}],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"31a26406-8149-11f0-9918-da7ad0900002","type":"pipelines","attributes":{"name":"fluent-pipeline","config":{"destinations":[{"id":"destination-1","inputs":["fluent-source-1"],"type":"datadog_logs"}],"processors":[],"sources":[{"id":"fluent-source-1","tls":{"ca_file":"/etc/ssl/certs/ca.crt","crt_file":"/etc/ssl/certs/fluent.crt","key_file":"/etc/ssl/private/fluent.key"},"type":"fluentd"}]}}},{"id":"47a53594-8149-11f0-9926-da7ad0900002","type":"pipelines","attributes":{"name":"syslogng-dest-pipeline","config":{"destinations":[{"id":"syslogng-destination-1","inputs":["source-1"],"keepalive":45000,"tls":{"crt_file":"/etc/certs/syslogng.crt"},"type":"syslog_ng"}],"processors":[],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"65424c18-8149-11f0-bb0a-da7ad0900002","type":"pipelines","attributes":{"name":"sentinelone + pipeline","config":{"destinations":[{"id":"sentinelone-dest-1","inputs":["source-1"],"region":"us","type":"sentinel_one"}],"processors":[],"sources":[{"id":"source-1","type":"datadog_agent"}]}}},{"id":"72abaaca-8149-11f0-9940-da7ad0900002","type":"pipelines","attributes":{"name":"socket-pipeline-udp","config":{"destinations":[{"id":"destination-1","inputs":["socket-source-2"],"type":"datadog_logs"}],"processors":[],"sources":[{"framing":{"delimiter":"|","method":"character_delimited"},"id":"socket-source-2","mode":"udp","type":"socket"}]}}},{"id":"4b51d94c-8165-11f0-ab16-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}},{"id":"d9ab0514-81a7-11f0-b0f7-da7ad0900002","type":"pipelines","attributes":{"name":"Main + Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}}],"meta":{"totalCount":440}} ' headers: @@ -70,7 +80,7 @@ interactions: accept: - '*/*' method: DELETE - uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/dd87c652-21fa-11f0-96e2-da7ad0900002 + uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/77329f9a-cae1-11f0-a2c9-da7ad0900002 response: body: string: '' diff --git a/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_bad_request_response.frozen b/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_bad_request_response.frozen index a8aa863f11..5020f3fdc8 100644 --- a/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_bad_request_response.frozen +++ b/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_bad_request_response.frozen @@ -1 +1 @@ -2025-04-25T17:29:34.672Z \ No newline at end of file +2025-11-26T16:03:39.454Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_bad_request_response.yaml b/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_bad_request_response.yaml index 6db97980e1..c481d966bf 100644 --- a/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_bad_request_response.yaml +++ b/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_bad_request_response.yaml @@ -11,7 +11,7 @@ interactions: uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines response: body: - string: '{"data":{"id":"deeea5f6-21fa-11f0-96e4-da7ad0900002","type":"pipelines","attributes":{"name":"Main + string: '{"data":{"id":"794d71d8-cae1-11f0-a2cb-da7ad0900002","type":"pipelines","attributes":{"name":"Main Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}}} ' @@ -22,7 +22,7 @@ interactions: code: 201 message: Created - request: - body: '{"data":{"attributes":{"config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"unknown-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Main + body: '{"data":{"attributes":{"config":{"destinations":[{"id":"datadog-logs-destination","inputs":["my-processor-group"],"type":"datadog_logs"}],"processors":[{"enabled":true,"id":"unknown-processor","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"enabled":true,"id":"filter-processor","include":"status:error","type":"filter"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Main Observability Pipeline"},"id":"3fa85f64-5717-4562-b3fc-2c963f66afa6","type":"pipelines"}}' headers: accept: @@ -30,11 +30,11 @@ interactions: content-type: - application/json method: PUT - uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/deeea5f6-21fa-11f0-96e4-da7ad0900002 + uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/794d71d8-cae1-11f0-a2cb-da7ad0900002 response: body: - string: '{"errors":[{"title":"Component with ID filter-processor is an unknown - component","meta":{"message":"Component with ID filter-processor is an unknown + string: '{"errors":[{"title":"Component with ID my-processor-group is an unknown + component","meta":{"message":"Component with ID my-processor-group is an unknown component"}},{"title":"The following components are unused: [datadog-agent-source unknown-processor]","meta":{"message":"The following components are unused: [datadog-agent-source unknown-processor]"}}]} @@ -52,7 +52,7 @@ interactions: accept: - '*/*' method: DELETE - uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/deeea5f6-21fa-11f0-96e4-da7ad0900002 + uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/794d71d8-cae1-11f0-a2cb-da7ad0900002 response: body: string: '' diff --git a/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_not_found_response.frozen b/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_not_found_response.frozen index 81821e325e..eb2e354e0e 100644 --- a/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_not_found_response.frozen +++ b/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_not_found_response.frozen @@ -1 +1 @@ -2025-04-25T17:29:36.564Z \ No newline at end of file +2025-11-26T16:03:41.901Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_not_found_response.yaml b/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_not_found_response.yaml index 7a8efada66..ce0b290bb9 100644 --- a/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_not_found_response.yaml +++ b/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_not_found_response.yaml @@ -1,6 +1,6 @@ interactions: - request: - body: '{"data":{"attributes":{"config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Main + body: '{"data":{"attributes":{"config":{"destinations":[{"id":"datadog-logs-destination","inputs":["my-processor-group"],"type":"datadog_logs"}],"processors":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"enabled":true,"id":"filter-processor","include":"status:error","type":"filter"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Main Observability Pipeline"},"id":"3fa85f64-5717-4562-b3fc-2c963f66afa6","type":"pipelines"}}' headers: accept: diff --git a/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_ok_response.frozen b/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_ok_response.frozen index 8195f92c72..d4844244d8 100644 --- a/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_ok_response.frozen +++ b/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_ok_response.frozen @@ -1 +1 @@ -2025-04-25T17:29:36.987Z \ No newline at end of file +2025-11-26T16:03:42.435Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_ok_response.yaml b/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_ok_response.yaml index 115d89402b..aab09b44e3 100644 --- a/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_ok_response.yaml +++ b/tests/v2/cassettes/test_scenarios/test_update_a_pipeline_returns_ok_response.yaml @@ -11,7 +11,7 @@ interactions: uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines response: body: - string: '{"data":{"id":"e04d5230-21fa-11f0-96e6-da7ad0900002","type":"pipelines","attributes":{"name":"Main + string: '{"data":{"id":"7b1707f4-cae1-11f0-a2cd-da7ad0900002","type":"pipelines","attributes":{"name":"Main Observability Pipeline","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}}} ' @@ -22,7 +22,7 @@ interactions: code: 201 message: Created - request: - body: '{"data":{"attributes":{"config":{"destinations":[{"id":"updated-datadog-logs-destination-id","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Updated + body: '{"data":{"attributes":{"config":{"destinations":[{"id":"updated-datadog-logs-destination-id","inputs":["my-processor-group"],"type":"datadog_logs"}],"processors":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"enabled":true,"id":"filter-processor","include":"status:error","type":"filter"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Updated Pipeline Name"},"id":"3fa85f64-5717-4562-b3fc-2c963f66afa6","type":"pipelines"}}' headers: accept: @@ -30,11 +30,11 @@ interactions: content-type: - application/json method: PUT - uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/e04d5230-21fa-11f0-96e6-da7ad0900002 + uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/7b1707f4-cae1-11f0-a2cd-da7ad0900002 response: body: - string: '{"data":{"id":"e04d5230-21fa-11f0-96e6-da7ad0900002","type":"pipelines","attributes":{"name":"Updated - Pipeline Name","config":{"destinations":[{"id":"updated-datadog-logs-destination-id","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}}} + string: '{"data":{"id":"7b1707f4-cae1-11f0-a2cd-da7ad0900002","type":"pipelines","attributes":{"name":"Updated + Pipeline Name","config":{"destinations":[{"id":"updated-datadog-logs-destination-id","inputs":["my-processor-group"],"type":"datadog_logs"}],"processors":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"enabled":true,"id":"filter-processor","include":"status:error","type":"filter"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}}} ' headers: @@ -49,7 +49,7 @@ interactions: accept: - '*/*' method: DELETE - uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/e04d5230-21fa-11f0-96e6-da7ad0900002 + uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/7b1707f4-cae1-11f0-a2cd-da7ad0900002 response: body: string: '' diff --git a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_bad_request_response.frozen b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_bad_request_response.frozen index f19e635b27..462334cc24 100644 --- a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_bad_request_response.frozen +++ b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_bad_request_response.frozen @@ -1 +1 @@ -2025-04-25T17:29:39.236Z \ No newline at end of file +2025-11-26T16:03:45.970Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_bad_request_response.yaml b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_bad_request_response.yaml index 7a8f9daae3..985ce2dbb2 100644 --- a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_bad_request_response.yaml +++ b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_bad_request_response.yaml @@ -1,6 +1,6 @@ interactions: - request: - body: '{"data":{"attributes":{"config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Main + body: '{"data":{"attributes":{"config":{"destinations":[{"id":"datadog-logs-destination","inputs":["my-processor-group"],"type":"datadog_logs"}],"processors":[{"enabled":true,"id":"my-processor-group","inputs":["datadog-agent-source"],"processors":[{"enabled":true,"id":"filter-processor","type":"filter"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Main Observability Pipeline"},"type":"pipelines"}}' headers: accept: @@ -11,7 +11,8 @@ interactions: uri: https://api.datadoghq.com/api/v2/remote_config/products/obs_pipelines/pipelines/validate response: body: - string: '{"errors":[{"title":"Field ''include'' is required","meta":{"field":"include","id":"filter-processor","message":"Field + string: '{"errors":[{"title":"Field ''include'' is required","meta":{"field":"include","id":"my-processor-group","message":"Field + ''include'' is required"}},{"title":"Field ''include'' is required","meta":{"field":"include","id":"filter-processor","message":"Field ''include'' is required"}}]} ' diff --git a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_ok_response.frozen b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_ok_response.frozen index c2e1e3c736..1d4961eb3d 100644 --- a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_ok_response.frozen +++ b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_ok_response.frozen @@ -1 +1 @@ -2025-04-25T17:29:39.613Z \ No newline at end of file +2025-11-26T16:03:46.484Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_ok_response.yaml b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_ok_response.yaml index e1cbf59d65..192d22320f 100644 --- a/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_ok_response.yaml +++ b/tests/v2/cassettes/test_scenarios/test_validate_an_observability_pipeline_returns_ok_response.yaml @@ -1,6 +1,6 @@ interactions: - request: - body: '{"data":{"attributes":{"config":{"destinations":[{"id":"datadog-logs-destination","inputs":["filter-processor"],"type":"datadog_logs"}],"processors":[{"id":"filter-processor","include":"service:my-service","inputs":["datadog-agent-source"],"type":"filter"}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Main + body: '{"data":{"attributes":{"config":{"destinations":[{"id":"datadog-logs-destination","inputs":["my-processor-group"],"type":"datadog_logs"}],"processors":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"enabled":true,"id":"filter-processor","include":"status:error","type":"filter"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Main Observability Pipeline"},"type":"pipelines"}}' headers: accept: @@ -16,7 +16,7 @@ interactions: ' headers: content-type: - - application/json + - application/vnd.api+json status: code: 200 message: OK diff --git a/tests/v2/features/observability_pipelines.feature b/tests/v2/features/observability_pipelines.feature index c7149b1e4f..c2a2e39f7c 100644 --- a/tests/v2/features/observability_pipelines.feature +++ b/tests/v2/features/observability_pipelines.feature @@ -12,7 +12,7 @@ Feature: Observability Pipelines Scenario: Create a new pipeline returns "Bad Request" response Given operation "CreatePipeline" enabled And new "CreatePipeline" request - And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["filter-processor"], "type": "datadog_logs"}], "processors": [{"id": "unknown-processor", "include": "service:my-service", "inputs": ["datadog-agent-source"], "type": "filter"}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "type": "pipelines"}} + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processors": [{"enabled": true, "id": "unknown-processor", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "type": "pipelines"}} When the request is sent Then the response status is 400 Bad Request @@ -20,7 +20,7 @@ Feature: Observability Pipelines Scenario: Create a new pipeline returns "Conflict" response Given operation "CreatePipeline" enabled And new "CreatePipeline" request - And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["filter-processor"], "type": "datadog_logs"}], "processors": [{"id": "filter-processor", "include": "service:my-service", "inputs": ["datadog-agent-source"], "type": "filter"}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "type": "pipelines"}} + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["filter-processor"], "type": "datadog_logs"}], "processors": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "type": "pipelines"}} When the request is sent Then the response status is 409 Conflict @@ -28,7 +28,7 @@ Feature: Observability Pipelines Scenario: Create a new pipeline returns "OK" response Given operation "CreatePipeline" enabled And new "CreatePipeline" request - And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["filter-processor"], "type": "datadog_logs"}], "processors": [{"id": "filter-processor", "include": "service:my-service", "inputs": ["datadog-agent-source"], "type": "filter"}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "type": "pipelines"}} + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processors": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "type": "pipelines"}} When the request is sent Then the response status is 201 OK And the response "data" has field "id" @@ -106,7 +106,7 @@ Feature: Observability Pipelines And new "UpdatePipeline" request And there is a valid "pipeline" in the system And request contains "pipeline_id" parameter from "pipeline.data.id" - And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["filter-processor"], "type": "datadog_logs"}], "processors": [{"id": "unknown-processor", "include": "service:my-service", "inputs": ["datadog-agent-source"], "type": "filter"}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", "type": "pipelines"}} + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processors": [{"enabled": true, "id": "unknown-processor", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", "type": "pipelines"}} When the request is sent Then the response status is 400 Bad Request @@ -115,7 +115,7 @@ Feature: Observability Pipelines Given operation "UpdatePipeline" enabled And new "UpdatePipeline" request And request contains "pipeline_id" parameter from "REPLACE.ME" - And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["filter-processor"], "type": "datadog_logs"}], "processors": [{"id": "filter-processor", "include": "service:my-service", "inputs": ["datadog-agent-source"], "type": "filter"}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", "type": "pipelines"}} + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["filter-processor"], "type": "datadog_logs"}], "processors": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", "type": "pipelines"}} When the request is sent Then the response status is 409 Conflict @@ -124,7 +124,7 @@ Feature: Observability Pipelines Given operation "UpdatePipeline" enabled And new "UpdatePipeline" request And request contains "pipeline_id" parameter with value "3fa85f64-5717-4562-b3fc-2c963f66afa6" - And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["filter-processor"], "type": "datadog_logs"}], "processors": [{"id": "filter-processor", "include": "service:my-service", "inputs": ["datadog-agent-source"], "type": "filter"}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", "type": "pipelines"}} + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processors": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", "type": "pipelines"}} When the request is sent Then the response status is 404 Not Found @@ -134,7 +134,7 @@ Feature: Observability Pipelines And there is a valid "pipeline" in the system And new "UpdatePipeline" request And request contains "pipeline_id" parameter from "pipeline.data.id" - And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "updated-datadog-logs-destination-id", "inputs": ["filter-processor"], "type": "datadog_logs"}], "processors": [{"id": "filter-processor", "include": "service:my-service", "inputs": ["datadog-agent-source"], "type": "filter"}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Updated Pipeline Name"}, "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", "type": "pipelines"}} + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "updated-datadog-logs-destination-id", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processors": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Updated Pipeline Name"}, "id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", "type": "pipelines"}} When the request is sent Then the response status is 200 OK And the response "data" has field "id" @@ -149,19 +149,19 @@ Feature: Observability Pipelines Scenario: Validate an observability pipeline returns "Bad Request" response Given operation "ValidatePipeline" enabled And new "ValidatePipeline" request - And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["filter-processor"], "type": "datadog_logs"}], "processors": [{"id": "filter-processor", "inputs": ["datadog-agent-source"], "type": "filter"}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "type": "pipelines"}} + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processors": [{"enabled": true, "id": "my-processor-group", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "type": "pipelines"}} When the request is sent Then the response status is 400 Bad Request And the response "errors[0].title" is equal to "Field 'include' is required" And the response "errors[0].meta.field" is equal to "include" - And the response "errors[0].meta.id" is equal to "filter-processor" + And the response "errors[0].meta.id" is equal to "my-processor-group" And the response "errors[0].meta.message" is equal to "Field 'include' is required" @team:DataDog/observability-pipelines Scenario: Validate an observability pipeline returns "OK" response Given operation "ValidatePipeline" enabled And new "ValidatePipeline" request - And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["filter-processor"], "type": "datadog_logs"}], "processors": [{"id": "filter-processor", "include": "service:my-service", "inputs": ["datadog-agent-source"], "type": "filter"}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "type": "pipelines"}} + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processors": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Main Observability Pipeline"}, "type": "pipelines"}} When the request is sent Then the response status is 200 OK And the response "errors" has length 0