Skip to content

Commit eda2053

Browse files
gabedosgabedos
authored
Fluentbit log configuration (#6)
1 parent e27d2ef commit eda2053

File tree

6 files changed

+245
-36
lines changed

6 files changed

+245
-36
lines changed

‎examples/ecs_fargate/main.tf‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,27 +6,35 @@ module "ecs_task" {
66
source = "../../modules/ecs_fargate"
77

88
# Configure Datadog
9-
dd_api_key = var.dd_api_key
10-
dd_api_key_secret_arn = var.dd_api_key_secret_arn
11-
dd_site = var.dd_site
12-
dd_service = var.dd_service
9+
dd_api_key = var.dd_api_key
10+
dd_api_key_secret_arn = var.dd_api_key_secret_arn
11+
dd_site = var.dd_site
12+
dd_service = var.dd_service
13+
dd_tags = "team:cont-p, owner:container-monitoring"
14+
dd_essential = true
15+
dd_is_datadog_dependency_enabled = true
1316

1417
dd_environment = [
1518
{
16-
name = "DD_TAGS",
17-
value = "team:cont-p, owner:container-monitoring"
19+
name = "DD_CUSTOM_ENV_VAR",
20+
value = "custom_value",
1821
},
1922
]
2023

2124
dd_dogstatsd = {
22-
dogstatsd_cardinality = "high",
25+
dogstatsd_cardinality = "high",
2326
origin_detection_enabled = true,
2427
}
2528

2629
dd_apm = {
2730
enabled = true,
2831
}
2932

33+
dd_log_collection = {
34+
enabled = true,
35+
is_log_router_dependency_enabled = true,
36+
}
37+
3038
# Configure Task Definition
3139
family = "datadog-terraform-app"
3240
container_definitions = {

‎examples/ecs_fargate/outputs.tf‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
output "example_module" {
2+
value = module.ecs_task
3+
}

‎modules/ecs_fargate/datadog.tf‎

Lines changed: 119 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,34 @@
11
locals {
2+
# Datadog Firelens log configuration
3+
dd_firelens_log_configuration = var.dd_log_collection.enabled ? merge(
4+
{
5+
logDriver = "awsfirelens"
6+
options = merge(
7+
{
8+
provider = "ecs"
9+
Name = "datadog"
10+
Host = "http-intake.logs.datadoghq.com"
11+
TLS = "on"
12+
retry_limit = "2"
13+
},
14+
var.dd_log_collection.log_driver_configuration.service_name != null ? { dd_service = var.dd_log_collection.log_driver_configuration.service_name } : {},
15+
var.dd_log_collection.log_driver_configuration.source_name != null ? { dd_source = var.dd_log_collection.log_driver_configuration.source_name } : {},
16+
var.dd_tags != null ? { dd_tags = var.dd_tags } : {},
17+
var.dd_api_key != null ? { apikey = var.dd_api_key } : {}
18+
)
19+
},
20+
var.dd_api_key_secret_arn != null ? {
21+
secretOptions = [
22+
{
23+
name = "apikey"
24+
valueFrom = var.dd_api_key_secret_arn
25+
}
26+
]
27+
} : {}
28+
) : null
29+
230
# Application container modifications
3-
is_linux = var.runtime_platform == null || var.runtime_platform.operating_system_family == null || var.runtime_platform.operating_system_family == "LINUX"
31+
is_linux = var.runtime_platform == null || var.runtime_platform.operating_system_family == null || var.runtime_platform.operating_system_family == "LINUX"
432
is_apm_socket_mount = var.dd_apm.enabled && var.dd_apm.socket_enabled && local.is_linux
533
is_dsd_socket_mount = var.dd_dogstatsd.enabled && var.dd_dogstatsd.socket_enabled && local.is_linux
634
is_apm_dsd_volume = local.is_apm_socket_mount || local.is_dsd_socket_mount
@@ -34,6 +62,20 @@ locals {
3462
}
3563
] : []
3664

65+
agent_dependency = var.dd_is_datadog_dependency_enabled && var.dd_health_check.command != null ? [
66+
{
67+
containerName = "datadog-agent"
68+
condition = "HEALTHY"
69+
}
70+
] : []
71+
72+
log_router_dependency = var.dd_log_collection.is_log_router_dependency_enabled && var.dd_log_collection.log_router_health_check.command != null && local.dd_firelens_log_configuration != null ? [
73+
{
74+
containerName = "datadog-log-router"
75+
condition = "HEALTHY"
76+
}
77+
] : []
78+
3779
modified_container_definitions = [
3880
for container in var.container_definitions : merge(
3981
container,
@@ -50,7 +92,14 @@ locals {
5092
lookup(container, "mountPoints", []),
5193
local.apm_dsd_mount
5294
)
53-
}
95+
dependsOn = concat(
96+
lookup(container, "dependsOn", []),
97+
local.agent_dependency,
98+
local.log_router_dependency,
99+
)
100+
},
101+
# Only override the log configuration if the Datadog firelens configuration exists
102+
local.dd_firelens_log_configuration != null ? { logConfiguration = local.dd_firelens_log_configuration } : {}
54103
)
55104
]
56105

@@ -74,8 +123,9 @@ locals {
74123
{ key = "DD_SERVICE", value = var.dd_service },
75124
{ key = "DD_ENV", value = var.dd_env },
76125
{ key = "DD_VERSION", value = var.dd_version },
77-
{ key = "DD_DOGSTATSD_TAG_CARDINALITY", value = var.dd_dogstatsd.dogstatsd_cardinality }
78-
# TODO: clusterName, ddTags, etc.
126+
{ key = "DD_DOGSTATSD_TAG_CARDINALITY", value = var.dd_dogstatsd.dogstatsd_cardinality },
127+
{ key = "DD_TAGS", value = var.dd_tags },
128+
{ key = "DD_CLUSTER_NAME", value = var.dd_cluster_name }
79129
] : { name = pair.key, value = pair.value } if pair.value != null
80130
]
81131

@@ -97,28 +147,74 @@ locals {
97147
)
98148

99149
# Datadog Agent container definition
100-
dd_agent_container = {
101-
name = "datadog-agent"
102-
image = "${var.dd_registry}:${var.dd_image_version}"
103-
environment = local.dd_agent_env
104-
secrets = var.dd_api_key_secret_arn != null ? [
150+
dd_agent_container = [
151+
merge(
105152
{
106-
name = "DD_API_KEY"
107-
valueFrom = var.dd_api_key_secret_arn
153+
name = "datadog-agent"
154+
image = "${var.dd_registry}:${var.dd_image_version}"
155+
essential = var.dd_essential
156+
environment = local.dd_agent_env
157+
secrets = var.dd_api_key_secret_arn != null ? [
158+
{
159+
name = "DD_API_KEY"
160+
valueFrom = var.dd_api_key_secret_arn
161+
}
162+
] : []
163+
portMappings = [
164+
{
165+
containerPort = 8125
166+
hostPort = 8125
167+
protocol = "udp"
168+
},
169+
{
170+
containerPort = 8126
171+
hostPort = 8126
172+
protocol = "tcp"
173+
}
174+
],
175+
mountPoints = local.apm_dsd_mount,
176+
logConfiguration = local.dd_firelens_log_configuration,
177+
dependsOn = var.dd_log_collection.is_log_router_dependency_enabled && local.dd_firelens_log_configuration != null ? local.log_router_dependency : [],
178+
},
179+
var.dd_health_check.command == null ? {} : {
180+
healthCheck = {
181+
command = var.dd_health_check.command
182+
interval = var.dd_health_check.interval
183+
timeout = var.dd_health_check.timeout
184+
retries = var.dd_health_check.retries
185+
startPeriod = var.dd_health_check.start_period
186+
}
108187
}
109-
] : []
110-
portMappings = [
188+
)
189+
]
190+
191+
# Datadog log router container definition
192+
dd_log_container = var.dd_log_collection.enabled ? [
193+
merge(
111194
{
112-
containerPort = 8125
113-
hostPort = 8125
114-
protocol = "udp"
195+
name = "datadog-log-router"
196+
image = "${var.dd_log_collection.registry}:${var.dd_log_collection.image_version}"
197+
essential = var.dd_log_collection.is_log_router_essential
198+
firelensConfiguration = {
199+
type = "fluentbit"
200+
options = {
201+
enable-ecs-log-metadata = "true"
202+
}
203+
}
204+
cpu = var.dd_log_collection.cpu
205+
memory_limit_mib = var.dd_log_collection.memory_limit_mib
206+
user = "0"
115207
},
116-
{
117-
containerPort = 8126
118-
hostPort = 8126
119-
protocol = "tcp"
208+
var.dd_log_collection.log_router_health_check.command == null ? {} : {
209+
healthCheck = {
210+
command = var.dd_log_collection.log_router_health_check.command
211+
interval = var.dd_log_collection.log_router_health_check.interval
212+
timeout = var.dd_log_collection.log_router_health_check.timeout
213+
retries = var.dd_log_collection.log_router_health_check.retries
214+
startPeriod = var.dd_log_collection.log_router_health_check.start_period
215+
}
120216
}
121-
],
122-
mountPoints = local.apm_dsd_mount
123-
}
217+
)
218+
] : []
219+
124220
}

‎modules/ecs_fargate/iam.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,8 @@ resource "aws_iam_role_policy_attachment" "new_role_dd_secret" {
8484
# in order to add permissions for the ecs_fargate check
8585

8686
locals {
87-
edit_task_role = var.task_role_arn != null
88-
create_task_role = var.task_role_arn == null
87+
edit_task_role = var.task_role_arn != null
88+
create_task_role = var.task_role_arn == null
8989
}
9090

9191
# ==============================

‎modules/ecs_fargate/main.tf‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ resource "aws_ecs_task_definition" "this" {
66

77
container_definitions = jsonencode(
88
concat(
9-
[local.dd_agent_container],
9+
local.dd_agent_container,
10+
local.dd_log_container,
1011
[for k, v in local.modified_container_definitions : v],
1112
)
1213
)
@@ -26,7 +27,7 @@ resource "aws_ecs_task_definition" "this" {
2627
# Prioritize the user-provided task execution role over the one created by the module
2728
execution_role_arn = var.execution_role_arn != null ? var.execution_role_arn : (length(aws_iam_role.new_ecs_task_execution_role) > 0 ? aws_iam_role.new_ecs_task_execution_role[0].arn : null)
2829

29-
family = var.family
30+
family = var.family
3031

3132
# Fargate incompatible parameter
3233
dynamic "inference_accelerator" {
@@ -74,7 +75,7 @@ resource "aws_ecs_task_definition" "this" {
7475
}
7576
}
7677

77-
skip_destroy = var.skip_destroy
78+
skip_destroy = var.skip_destroy
7879
# Prioritize the user-provided task role over the one created by the module
7980
task_role_arn = var.task_role_arn != null ? var.task_role_arn : (length(aws_iam_role.new_ecs_task_role) > 0 ? aws_iam_role.new_ecs_task_role[0].arn : null)
8081

‎modules/ecs_fargate/variables.tf‎

Lines changed: 102 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,36 @@ variable "dd_image_version" {
2626
default = "latest"
2727
}
2828

29+
variable "dd_essential" {
30+
description = "Whether the Datadog Agent container is essential"
31+
type = bool
32+
default = false
33+
}
34+
35+
variable "dd_is_datadog_dependency_enabled" {
36+
description = "Whether the Datadog Agent container is a dependency for other containers"
37+
type = bool
38+
default = false
39+
}
40+
41+
variable "dd_health_check" {
42+
description = "Datadog Agent health check configuration"
43+
type = object({
44+
command = optional(list(string))
45+
interval = optional(number)
46+
retries = optional(number)
47+
start_period = optional(number)
48+
timeout = optional(number)
49+
})
50+
default = {
51+
command = ["CMD-SHELL", "/probe.sh"]
52+
interval = 15
53+
retries = 3
54+
start_period = 60
55+
timeout = 5
56+
}
57+
}
58+
2959
variable "dd_site" {
3060
description = "Datadog Site"
3161
type = string
@@ -38,6 +68,18 @@ variable "dd_environment" {
3868
default = [{}]
3969
}
4070

71+
variable "dd_tags" {
72+
description = "Datadog Agent global tags (eg. `key1:value1, key2:value2`)"
73+
type = string
74+
default = null
75+
}
76+
77+
variable "dd_cluster_name" {
78+
description = "Datadog cluster name."
79+
type = string
80+
default = null
81+
}
82+
4183
variable "dd_service" {
4284
description = "The task service name. Used for tagging (UST)"
4385
type = string
@@ -75,7 +117,7 @@ variable "dd_dogstatsd" {
75117
error_message = "The Datadog Dogstatsd configuration must be defined."
76118
}
77119
validation {
78-
condition = var.dd_dogstatsd.dogstatsd_cardinality == "low" || var.dd_dogstatsd.dogstatsd_cardinality == "orchestrator" || var.dd_dogstatsd.dogstatsd_cardinality == "high"
120+
condition = var.dd_dogstatsd.dogstatsd_cardinality == "low" || var.dd_dogstatsd.dogstatsd_cardinality == "orchestrator" || var.dd_dogstatsd.dogstatsd_cardinality == "high"
79121
error_message = "The Datadog Dogstatsd cardinality must be one of 'low', 'orchestrator', or 'high'."
80122
}
81123
}
@@ -96,6 +138,65 @@ variable "dd_apm" {
96138
}
97139
}
98140

141+
variable "dd_log_collection" {
142+
description = "Configuration for Datadog Log Collection"
143+
type = object({
144+
enabled = optional(bool, true)
145+
registry = optional(string, "public.ecr.aws/aws-observability/aws-for-fluent-bit")
146+
image_version = optional(string, "stable")
147+
cpu = optional(number)
148+
memory_limit_mib = optional(number)
149+
is_log_router_essential = optional(bool, false)
150+
is_log_router_dependency_enabled = optional(bool, false)
151+
log_router_health_check = optional(object({
152+
command = optional(list(string))
153+
interval = optional(number)
154+
retries = optional(number)
155+
start_period = optional(number)
156+
timeout = optional(number)
157+
}),
158+
{
159+
command = ["CMD-SHELL", "exit 0"]
160+
interval = 5
161+
retries = 3
162+
start_period = 15
163+
timeout = 5
164+
}
165+
)
166+
log_driver_configuration = optional(object({
167+
host_endpoint = optional(string, "http-intake.logs.datadoghq.com")
168+
tls = optional(bool)
169+
compress = optional(string)
170+
service_name = optional(string)
171+
source_name = optional(string)
172+
message_key = optional(string)
173+
}),
174+
{
175+
host_endpoint = "http-intake.logs.datadoghq.com"
176+
}
177+
)
178+
})
179+
default = {
180+
enabled = false
181+
is_log_router_essential = false
182+
log_driver_configuration = {
183+
host_endpoint = "http-intake.logs.datadoghq.com"
184+
}
185+
}
186+
validation {
187+
condition = var.dd_log_collection != null
188+
error_message = "The Datadog Log Collection configuration must be defined."
189+
}
190+
validation {
191+
condition = var.dd_log_collection.log_driver_configuration != null
192+
error_message = "The Datadog Log Collection log driver configuration must be defined."
193+
}
194+
validation {
195+
condition = var.dd_log_collection.log_driver_configuration.host_endpoint != null
196+
error_message = "The Datadog Log Collection log driver configuration host endpoint must be defined."
197+
}
198+
}
199+
99200
################################################################################
100201
# Task Definition
101202
################################################################################

0 commit comments

Comments
 (0)