[CONTP-951] Support env vars passed down to dd logging container (#41)

* chore: add task family name var in example for flexibility

* feat: add environment option to dd_log_collector

* test: add envvar to logging only smoke test

* chore: update module docs

* chore: add log environment safety checks

Author: Mathew-Estafanous

.gitignore

@@ -45,3 +45,6 @@ target
 bin
 obj
 .DS_Store
+
+# Jetbrains
+.idea

examples/ecs_fargate/README.md

@@ -9,6 +9,7 @@ This example showcases a simple ECS Fargate Task Definition with out of the box
   * Set the `dd_api_key` to the Datadog API Key (required)
   * Set the `dd_service` to the name of the service you want to use to filter for the resource in Datadog
   * Set the `dd_site` to the [Datadog destination site](https://docs.datadoghq.com/getting_started/site/) for your metrics, traces, and logs
+  * (Optional) Set `task_family_name` to the name of the task family (default: "dummy-terraform-app")
 * Run the following commands:
 
 ```bash

examples/ecs_fargate/main.tf

@@ -45,7 +45,7 @@ module "datadog_ecs_fargate_task" {
   }
 
   # Configure Task Definition
-  family = "dummy-terraform-app"
+  family = var.task_family_name
   container_definitions = jsonencode([
     {
       name      = "dummy-dogstatsd-app",

examples/ecs_fargate/variables.tf

@@ -26,3 +26,9 @@ variable "dd_site" {
   type        = string
   default     = "datadoghq.com"
 }
+
+variable "task_family_name" {
+  description = "The ECS task family name"
+  type        = string
+  default     = "dummy-terraform-app"
+}

modules/ecs_fargate/README.md

@@ -210,8 +210,6 @@ No modules.
 | [aws_iam_role_policy_attachment.new_role_ecs_task_permissions](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_policy_document.dd_ecs_task_permissions](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.dd_secret_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
-| [aws_iam_role.ecs_task_exec_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source |
-| [aws_iam_role.ecs_task_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source |
 
 ## Inputs
 
@@ -233,7 +231,7 @@ No modules.
 | <a name="input_dd_health_check"></a> [dd\_health\_check](#input\_dd\_health\_check) | Datadog Agent health check configuration | <pre>object({<br/>    command      = optional(list(string))<br/>    interval     = optional(number)<br/>    retries      = optional(number)<br/>    start_period = optional(number)<br/>    timeout      = optional(number)<br/>  })</pre> | <pre>{<br/>  "command": [<br/>    "CMD-SHELL",<br/>    "/probe.sh"<br/>  ],<br/>  "interval": 15,<br/>  "retries": 3,<br/>  "start_period": 60,<br/>  "timeout": 5<br/>}</pre> | no |
 | <a name="input_dd_image_version"></a> [dd\_image\_version](#input\_dd\_image\_version) | Datadog Agent image version | `string` | `"latest"` | no |
 | <a name="input_dd_is_datadog_dependency_enabled"></a> [dd\_is\_datadog\_dependency\_enabled](#input\_dd\_is\_datadog\_dependency\_enabled) | Whether the Datadog Agent container is a dependency for other containers | `bool` | `false` | no |
-| <a name="input_dd_log_collection"></a> [dd\_log\_collection](#input\_dd\_log\_collection) | Configuration for Datadog Log Collection | <pre>object({<br/>    enabled = optional(bool, false)<br/>    fluentbit_config = optional(object({<br/>      registry                         = optional(string, "public.ecr.aws/aws-observability/aws-for-fluent-bit")<br/>      image_version                    = optional(string, "stable")<br/>      cpu                              = optional(number)<br/>      memory_limit_mib                 = optional(number)<br/>      is_log_router_essential          = optional(bool, false)<br/>      is_log_router_dependency_enabled = optional(bool, false)<br/>      log_router_health_check = optional(object({<br/>        command      = optional(list(string))<br/>        interval     = optional(number)<br/>        retries      = optional(number)<br/>        start_period = optional(number)<br/>        timeout      = optional(number)<br/>        }),<br/>        {<br/>          command      = ["CMD-SHELL", "exit 0"]<br/>          interval     = 5<br/>          retries      = 3<br/>          start_period = 15<br/>          timeout      = 5<br/>        }<br/>      )<br/>      firelens_options = optional(object({<br/>        config_file_type  = optional(string)<br/>        config_file_value = optional(string)<br/>      }))<br/>      log_driver_configuration = optional(object({<br/>        host_endpoint = optional(string, "http-intake.logs.datadoghq.com")<br/>        tls           = optional(bool)<br/>        compress      = optional(string)<br/>        service_name  = optional(string)<br/>        source_name   = optional(string)<br/>        message_key   = optional(string)<br/>        }),<br/>        {<br/>          host_endpoint = "http-intake.logs.datadoghq.com"<br/>        }<br/>      )<br/>      }),<br/>      {<br/>        fluentbit_config = {<br/>          registry      = "public.ecr.aws/aws-observability/aws-for-fluent-bit"<br/>          image_version = "stable"<br/>          log_driver_configuration = {<br/>            host_endpoint = "http-intake.logs.datadoghq.com"<br/>          }<br/>        }<br/>      }<br/>    )<br/>  })</pre> | <pre>{<br/>  "enabled": false,<br/>  "fluentbit_config": {<br/>    "is_log_router_essential": false,<br/>    "log_driver_configuration": {<br/>      "host_endpoint": "http-intake.logs.datadoghq.com"<br/>    }<br/>  }<br/>}</pre> | no |
+| <a name="input_dd_log_collection"></a> [dd\_log\_collection](#input\_dd\_log\_collection) | Configuration for Datadog Log Collection | <pre>object({<br/>    enabled = optional(bool, false)<br/>    fluentbit_config = optional(object({<br/>      registry                         = optional(string, "public.ecr.aws/aws-observability/aws-for-fluent-bit")<br/>      image_version                    = optional(string, "stable")<br/>      cpu                              = optional(number)<br/>      memory_limit_mib                 = optional(number)<br/>      is_log_router_essential          = optional(bool, false)<br/>      is_log_router_dependency_enabled = optional(bool, false)<br/>      environment                      = optional(list(map(string)), [{}])<br/>      log_router_health_check = optional(object({<br/>        command      = optional(list(string))<br/>        interval     = optional(number)<br/>        retries      = optional(number)<br/>        start_period = optional(number)<br/>        timeout      = optional(number)<br/>        }),<br/>        {<br/>          command      = ["CMD-SHELL", "exit 0"]<br/>          interval     = 5<br/>          retries      = 3<br/>          start_period = 15<br/>          timeout      = 5<br/>        }<br/>      )<br/>      firelens_options = optional(object({<br/>        config_file_type  = optional(string)<br/>        config_file_value = optional(string)<br/>      }))<br/>      log_driver_configuration = optional(object({<br/>        host_endpoint = optional(string, "http-intake.logs.datadoghq.com")<br/>        tls           = optional(bool)<br/>        compress      = optional(string)<br/>        service_name  = optional(string)<br/>        source_name   = optional(string)<br/>        message_key   = optional(string)<br/>        }),<br/>        {<br/>          host_endpoint = "http-intake.logs.datadoghq.com"<br/>        }<br/>      )<br/>      }),<br/>      {<br/>        fluentbit_config = {<br/>          registry      = "public.ecr.aws/aws-observability/aws-for-fluent-bit"<br/>          image_version = "stable"<br/>          log_driver_configuration = {<br/>            host_endpoint = "http-intake.logs.datadoghq.com"<br/>          }<br/>        }<br/>      }<br/>    )<br/>  })</pre> | <pre>{<br/>  "enabled": false,<br/>  "fluentbit_config": {<br/>    "is_log_router_essential": false,<br/>    "log_driver_configuration": {<br/>      "host_endpoint": "http-intake.logs.datadoghq.com"<br/>    }<br/>  }<br/>}</pre> | no |
 | <a name="input_dd_memory_limit_mib"></a> [dd\_memory\_limit\_mib](#input\_dd\_memory\_limit\_mib) | Datadog Agent container memory limit in MiB | `number` | `null` | no |
 | <a name="input_dd_registry"></a> [dd\_registry](#input\_dd\_registry) | Datadog Agent image registry | `string` | `"public.ecr.aws/datadog/agent"` | no |
 | <a name="input_dd_service"></a> [dd\_service](#input\_dd\_service) | The task service name. Used for tagging (UST) | `string` | `null` | no |

modules/ecs_fargate/datadog.tf

@@ -338,6 +338,13 @@ locals {
     )
   ]
 
+  dd_log_environment = var.dd_log_collection.fluentbit_config.environment != null ? var.dd_log_collection.fluentbit_config.environment : []
+
+  dd_log_agent_env = concat(
+    local.ust_env_vars,
+    local.dd_log_environment
+  )
+
   # Datadog log router container definition
   dd_log_container = local.is_fluentbit_supported ? [
     merge(
@@ -359,7 +366,7 @@ locals {
         memory_limit_mib = var.dd_log_collection.fluentbit_config.memory_limit_mib
         user             = "0"
         mountPoints      = []
-        environment      = local.ust_env_vars
+        environment      = local.dd_log_agent_env
         portMappings     = []
         systemControls   = []
         volumesFrom      = []

modules/ecs_fargate/variables.tf

@@ -191,6 +191,7 @@ variable "dd_log_collection" {
       memory_limit_mib                 = optional(number)
       is_log_router_essential          = optional(bool, false)
       is_log_router_dependency_enabled = optional(bool, false)
+      environment                      = optional(list(map(string)), [{}])
       log_router_health_check = optional(object({
         command      = optional(list(string))
         interval     = optional(number)

smoke_tests/ecs_fargate/logging-only.tf

@@ -31,6 +31,12 @@ module "dd_task_logging_only" {
         config_file_type  = "file"
         config_file_value = "file:///fluent-bit/etc/fluent-bit.conf"
       }
+      environment = [
+        {
+          name  = "DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL"
+          value = "true"
+        }
+      ]
     }
   }
 

tests/logging_only_test.go

@@ -87,7 +87,8 @@ func (s *ECSFargateSuite) TestLoggingOnly() {
 
 	// Verify log router environment variables
 	expectedLogRouterEnvVars := map[string]string{
-		"DD_SERVICE": "test-service",
+		"DD_SERVICE":                           "test-service",
+		"DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL": "true",
 	}
 	AssertEnvVars(s.T(), logRouterContainer, expectedLogRouterEnvVars)