Skip to content

WIP Add processors groups to Observability Pipelines #2929

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

WIP Add processors groups to Observability Pipelines #2929

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
330 changes: 161 additions & 169 deletions .generator/schemas/v2/openapi.yaml
View file
Open in desktop

Large diffs are not rendered by default.

7 changes: 7 additions & 0 deletions docs/datadog_api_client.v2.model.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14823,6 +14823,13 @@ datadog\_api\_client.v2.model.observability\_pipeline\_config\_destination\_item
:members:
:show-inheritance:

datadog\_api\_client.v2.model.observability\_pipeline\_config\_processor\_group module
--------------------------------------------------------------------------------------

.. automodule:: datadog_api_client.v2.model.observability_pipeline_config_processor_group
:members:
:show-inheritance:

datadog\_api\_client.v2.model.observability\_pipeline\_config\_processor\_item module
-------------------------------------------------------------------------------------

Expand Down
19 changes: 15 additions & 4 deletions examples/v2/observability-pipelines/CreatePipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
from datadog_api_client import ApiClient, Configuration
from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi
from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig
from datadog_api_client.v2.model.observability_pipeline_config_processor_group import (
ObservabilityPipelineConfigProcessorGroup,
)
from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes
from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import (
ObservabilityPipelineDatadogAgentSource,
Expand Down Expand Up @@ -33,19 +36,27 @@
ObservabilityPipelineDatadogLogsDestination(
id="datadog-logs-destination",
inputs=[
"filter-processor",
"my-processor-group",
],
type=ObservabilityPipelineDatadogLogsDestinationType.DATADOG_LOGS,
),
],
processors=[
ObservabilityPipelineFilterProcessor(
id="filter-processor",
ObservabilityPipelineConfigProcessorGroup(
enabled=True,
id="my-processor-group",
include="service:my-service",
inputs=[
"datadog-agent-source",
],
type=ObservabilityPipelineFilterProcessorType.FILTER,
processors=[
ObservabilityPipelineFilterProcessor(
enabled=True,
id="filter-processor",
include="status:error",
type=ObservabilityPipelineFilterProcessorType.FILTER,
),
],
),
],
sources=[
Expand Down
19 changes: 15 additions & 4 deletions examples/v2/observability-pipelines/UpdatePipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@
from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi
from datadog_api_client.v2.model.observability_pipeline import ObservabilityPipeline
from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig
from datadog_api_client.v2.model.observability_pipeline_config_processor_group import (
ObservabilityPipelineConfigProcessorGroup,
)
from datadog_api_client.v2.model.observability_pipeline_data import ObservabilityPipelineData
from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes
from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import (
Expand Down Expand Up @@ -37,19 +40,27 @@
ObservabilityPipelineDatadogLogsDestination(
id="updated-datadog-logs-destination-id",
inputs=[
"filter-processor",
"my-processor-group",
],
type=ObservabilityPipelineDatadogLogsDestinationType.DATADOG_LOGS,
),
],
processors=[
ObservabilityPipelineFilterProcessor(
id="filter-processor",
ObservabilityPipelineConfigProcessorGroup(
enabled=True,
id="my-processor-group",
include="service:my-service",
inputs=[
"datadog-agent-source",
],
type=ObservabilityPipelineFilterProcessorType.FILTER,
processors=[
ObservabilityPipelineFilterProcessor(
enabled=True,
id="filter-processor",
include="status:error",
type=ObservabilityPipelineFilterProcessorType.FILTER,
),
],
),
],
sources=[
Expand Down
19 changes: 15 additions & 4 deletions examples/v2/observability-pipelines/ValidatePipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
from datadog_api_client import ApiClient, Configuration
from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi
from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig
from datadog_api_client.v2.model.observability_pipeline_config_processor_group import (
ObservabilityPipelineConfigProcessorGroup,
)
from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes
from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import (
ObservabilityPipelineDatadogAgentSource,
Expand Down Expand Up @@ -33,19 +36,27 @@
ObservabilityPipelineDatadogLogsDestination(
id="datadog-logs-destination",
inputs=[
"filter-processor",
"my-processor-group",
],
type=ObservabilityPipelineDatadogLogsDestinationType.DATADOG_LOGS,
),
],
processors=[
ObservabilityPipelineFilterProcessor(
id="filter-processor",
ObservabilityPipelineConfigProcessorGroup(
enabled=True,
id="my-processor-group",
include="service:my-service",
inputs=[
"datadog-agent-source",
],
type=ObservabilityPipelineFilterProcessorType.FILTER,
processors=[
ObservabilityPipelineFilterProcessor(
enabled=True,
id="filter-processor",
include="status:error",
type=ObservabilityPipelineFilterProcessorType.FILTER,
),
],
),
],
sources=[
Expand Down
14 changes: 7 additions & 7 deletions src/datadog_api_client/v2/model/observability_pipeline_add_env_vars_processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,42 +31,42 @@ def openapi_types(_):
)

return {
"enabled": (bool,),
"id": (str,),
"include": (str,),
"inputs": ([str],),
"type": (ObservabilityPipelineAddEnvVarsProcessorType,),
"variables": ([ObservabilityPipelineAddEnvVarsProcessorVariable],),
}

attribute_map = {
"enabled": "enabled",
"id": "id",
"include": "include",
"inputs": "inputs",
"type": "type",
"variables": "variables",
}

def __init__(
self_,
enabled: bool,
id: str,
include: str,
inputs: List[str],
type: ObservabilityPipelineAddEnvVarsProcessorType,
variables: List[ObservabilityPipelineAddEnvVarsProcessorVariable],
**kwargs,
):
"""
The ``add_env_vars`` processor adds environment variable values to log events.

:param enabled: Whether this processor is enabled.
:type enabled: bool

:param id: The unique identifier for this component. Used to reference this processor in the pipeline.
:type id: str

:param include: A Datadog search query used to determine which logs this processor targets.
:type include: str

:param inputs: A list of component IDs whose output is used as the input for this processor.
:type inputs: [str]

:param type: The processor type. The value should always be ``add_env_vars``.
:type type: ObservabilityPipelineAddEnvVarsProcessorType

Expand All @@ -75,8 +75,8 @@ def __init__(
"""
super().__init__(kwargs)

self_.enabled = enabled
self_.id = id
self_.include = include
self_.inputs = inputs
self_.type = type
self_.variables = variables
14 changes: 7 additions & 7 deletions src/datadog_api_client/v2/model/observability_pipeline_add_fields_processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,33 +27,36 @@ def openapi_types(_):
)

return {
"enabled": (bool,),
"fields": ([ObservabilityPipelineFieldValue],),
"id": (str,),
"include": (str,),
"inputs": ([str],),
"type": (ObservabilityPipelineAddFieldsProcessorType,),
}

attribute_map = {
"enabled": "enabled",
"fields": "fields",
"id": "id",
"include": "include",
"inputs": "inputs",
"type": "type",
}

def __init__(
self_,
enabled: bool,
fields: List[ObservabilityPipelineFieldValue],
id: str,
include: str,
inputs: List[str],
type: ObservabilityPipelineAddFieldsProcessorType,
**kwargs,
):
"""
The ``add_fields`` processor adds static key-value fields to logs.

:param enabled: Whether this processor is enabled.
:type enabled: bool

:param fields: A list of static fields (key-value pairs) that is added to each log event processed by this component.
:type fields: [ObservabilityPipelineFieldValue]

Expand All @@ -63,16 +66,13 @@ def __init__(
:param include: A Datadog search query used to determine which logs this processor targets.
:type include: str

:param inputs: A list of component IDs whose output is used as the ``input`` for this component.
:type inputs: [str]

:param type: The processor type. The value should always be ``add_fields``.
:type type: ObservabilityPipelineAddFieldsProcessorType
"""
super().__init__(kwargs)

self_.enabled = enabled
self_.fields = fields
self_.id = id
self_.include = include
self_.inputs = inputs
self_.type = type
83 changes: 8 additions & 75 deletions src/datadog_api_client/v2/model/observability_pipeline_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
from datadog_api_client.v2.model.observability_pipeline_config_destination_item import (
ObservabilityPipelineConfigDestinationItem,
)
from datadog_api_client.v2.model.observability_pipeline_config_processor_item import (
ObservabilityPipelineConfigProcessorItem,
from datadog_api_client.v2.model.observability_pipeline_config_processor_group import (
ObservabilityPipelineConfigProcessorGroup,
)
from datadog_api_client.v2.model.observability_pipeline_config_source_item import (
ObservabilityPipelineConfigSourceItem,
Expand Down Expand Up @@ -76,48 +76,6 @@
from datadog_api_client.v2.model.observability_pipeline_google_pub_sub_destination import (
ObservabilityPipelineGooglePubSubDestination,
)
from datadog_api_client.v2.model.observability_pipeline_filter_processor import ObservabilityPipelineFilterProcessor
from datadog_api_client.v2.model.observability_pipeline_parse_json_processor import (
ObservabilityPipelineParseJSONProcessor,
)
from datadog_api_client.v2.model.observability_pipeline_quota_processor import ObservabilityPipelineQuotaProcessor
from datadog_api_client.v2.model.observability_pipeline_add_fields_processor import (
ObservabilityPipelineAddFieldsProcessor,
)
from datadog_api_client.v2.model.observability_pipeline_remove_fields_processor import (
ObservabilityPipelineRemoveFieldsProcessor,
)
from datadog_api_client.v2.model.observability_pipeline_rename_fields_processor import (
ObservabilityPipelineRenameFieldsProcessor,
)
from datadog_api_client.v2.model.observability_pipeline_generate_metrics_processor import (
ObservabilityPipelineGenerateMetricsProcessor,
)
from datadog_api_client.v2.model.observability_pipeline_sample_processor import ObservabilityPipelineSampleProcessor
from datadog_api_client.v2.model.observability_pipeline_parse_grok_processor import (
ObservabilityPipelineParseGrokProcessor,
)
from datadog_api_client.v2.model.observability_pipeline_sensitive_data_scanner_processor import (
ObservabilityPipelineSensitiveDataScannerProcessor,
)
from datadog_api_client.v2.model.observability_pipeline_ocsf_mapper_processor import (
ObservabilityPipelineOcsfMapperProcessor,
)
from datadog_api_client.v2.model.observability_pipeline_add_env_vars_processor import (
ObservabilityPipelineAddEnvVarsProcessor,
)
from datadog_api_client.v2.model.observability_pipeline_dedupe_processor import ObservabilityPipelineDedupeProcessor
from datadog_api_client.v2.model.observability_pipeline_enrichment_table_processor import (
ObservabilityPipelineEnrichmentTableProcessor,
)
from datadog_api_client.v2.model.observability_pipeline_reduce_processor import ObservabilityPipelineReduceProcessor
from datadog_api_client.v2.model.observability_pipeline_throttle_processor import (
ObservabilityPipelineThrottleProcessor,
)
from datadog_api_client.v2.model.observability_pipeline_custom_processor import ObservabilityPipelineCustomProcessor
from datadog_api_client.v2.model.observability_pipeline_datadog_tags_processor import (
ObservabilityPipelineDatadogTagsProcessor,
)
from datadog_api_client.v2.model.observability_pipeline_kafka_source import ObservabilityPipelineKafkaSource
from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import (
ObservabilityPipelineDatadogAgentSource,
Expand Down Expand Up @@ -160,16 +118,16 @@ def openapi_types(_):
from datadog_api_client.v2.model.observability_pipeline_config_destination_item import (
ObservabilityPipelineConfigDestinationItem,
)
from datadog_api_client.v2.model.observability_pipeline_config_processor_item import (
ObservabilityPipelineConfigProcessorItem,
from datadog_api_client.v2.model.observability_pipeline_config_processor_group import (
ObservabilityPipelineConfigProcessorGroup,
)
from datadog_api_client.v2.model.observability_pipeline_config_source_item import (
ObservabilityPipelineConfigSourceItem,
)

return {
"destinations": ([ObservabilityPipelineConfigDestinationItem],),
"processors": ([ObservabilityPipelineConfigProcessorItem],),
"processors": ([ObservabilityPipelineConfigProcessorGroup],),
"sources": ([ObservabilityPipelineConfigSourceItem],),
}

Expand Down Expand Up @@ -226,32 +184,7 @@ def __init__(
ObservabilityPipelineSocketSource,
]
],
processors: Union[
List[
Union[
ObservabilityPipelineConfigProcessorItem,
ObservabilityPipelineFilterProcessor,
ObservabilityPipelineParseJSONProcessor,
ObservabilityPipelineQuotaProcessor,
ObservabilityPipelineAddFieldsProcessor,
ObservabilityPipelineRemoveFieldsProcessor,
ObservabilityPipelineRenameFieldsProcessor,
ObservabilityPipelineGenerateMetricsProcessor,
ObservabilityPipelineSampleProcessor,
ObservabilityPipelineParseGrokProcessor,
ObservabilityPipelineSensitiveDataScannerProcessor,
ObservabilityPipelineOcsfMapperProcessor,
ObservabilityPipelineAddEnvVarsProcessor,
ObservabilityPipelineDedupeProcessor,
ObservabilityPipelineEnrichmentTableProcessor,
ObservabilityPipelineReduceProcessor,
ObservabilityPipelineThrottleProcessor,
ObservabilityPipelineCustomProcessor,
ObservabilityPipelineDatadogTagsProcessor,
]
],
UnsetType,
] = unset,
processors: Union[List[ObservabilityPipelineConfigProcessorGroup], UnsetType] = unset,
**kwargs,
):
"""
Expand All @@ -260,8 +193,8 @@ def __init__(
:param destinations: A list of destination components where processed logs are sent.
:type destinations: [ObservabilityPipelineConfigDestinationItem]

:param processors: A list of processors that transform or enrich log data.
:type processors: [ObservabilityPipelineConfigProcessorItem], optional
:param processors: A list of processor groups that transform or enrich log data.
:type processors: [ObservabilityPipelineConfigProcessorGroup], optional

:param sources: A list of configured data sources for the pipeline.
:type sources: [ObservabilityPipelineConfigSourceItem]
Expand Down
Loading
Loading