fix s3control endpoint override and respect config for mwaa (#91)

Author: bentsku

README.md

@@ -77,6 +77,7 @@ ADDITIONAL_TF_OVERRIDE_LOCATIONS=/path/to/module1,path/to/module2 tflocal plan
 
 ## Change Log
 
+* v0.25.0: Improve `s3control` local endpoint override and respect `AWS_ENDPOINT_URL` configuration for `mwaa`
 * v0.24.1: Exclude broken `python-hcl2` version from requirements
 * v0.24.0: Add support to return `terraform-local` version when calling `tflocal -version` and fix AWS provider detection
 * v0.23.1: Fix endpoint overrides for Terraform AWS provider >= 6.0.0-beta2

bin/tflocal

@@ -35,6 +35,7 @@ CUSTOMIZE_ACCESS_KEY = str(os.environ.get("CUSTOMIZE_ACCESS_KEY")).strip().lower
     "1",
     "true",
 ]
+LOCALHOST = "localhost"
 LOCALHOST_HOSTNAME = "localhost.localstack.cloud"
 S3_HOSTNAME = os.environ.get("S3_HOSTNAME") or f"s3.{LOCALHOST_HOSTNAME}"
 USE_EXEC = str(os.environ.get("USE_EXEC")).strip().lower() in ["1", "true"]
@@ -53,7 +54,7 @@ LS_PROVIDERS_FILE = (
 LOCALSTACK_HOSTNAME = (
     urlparse(AWS_ENDPOINT_URL).hostname
     or os.environ.get("LOCALSTACK_HOSTNAME")
-    or "localhost"
+    or LOCALHOST
 )
 EDGE_PORT = int(urlparse(AWS_ENDPOINT_URL).port or os.environ.get("EDGE_PORT") or 4566)
 AWS_PROVIDER_NAME_SUFFIX = "/hashicorp/aws"
@@ -541,10 +542,16 @@ def get_service_endpoint(service: str) -> str:
 
     # some services need specific hostnames
     hostname = LOCALSTACK_HOSTNAME
+    # if the user has not set the LOCALSTACK_HOSTNAME, and it fell back to `localhost`, we must force the endpoint to
+    # be make sure it can resolve subdomains
+    subdomain_compatible_endpoint = LOCALHOST_HOSTNAME if hostname == LOCALHOST else hostname
     if service == "s3":
         hostname = S3_HOSTNAME
     elif service == "mwaa":
-        hostname = f"mwaa.{LOCALHOST_HOSTNAME}"
+        hostname = f"mwaa.{subdomain_compatible_endpoint}"
+    elif service == "s3control":
+        # `s3control` sets the account-id as part of the subdomain of the endpoint
+        hostname = subdomain_compatible_endpoint
 
     return f"http://{hostname}:{EDGE_PORT}"
 

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = terraform-local
-version = 0.24.1
+version = 0.25.0
 url = https://github.com/localstack/terraform-local
 author = LocalStack Team
 author_email = info@localstack.cloud

tests/test_apply.py

@@ -409,7 +409,6 @@ def test_versioned_endpoints(monkeypatch, provider_version):
       secret_key               = "test"
       skip_credentials_validation = true
       skip_metadata_api_check   = true
-      skip_requesting_account_id = true
       endpoints {
         sns = "http://localhost:4566"
       }
@@ -444,6 +443,71 @@ def test_versioned_endpoints(monkeypatch, provider_version):
                 assert "iotevents" not in endpoints
 
 
+@pytest.mark.parametrize("endpoint_host", ["", "test-host"])
+def test_subdomain_endpoints(monkeypatch, endpoint_host):
+    # we are using the `AWS_ENDPOINT_URL`, but setting the `LOCALSTACK_HOSTNAME` and `EDGE_PORT` (both deprecated)
+    # would have the same behavior
+    aws_endpoint_url = f"http://{endpoint_host}:4566" if endpoint_host else ""
+    monkeypatch.setenv("AWS_ENDPOINT_URL", aws_endpoint_url)
+    bucket_name = f"bucket-{short_uid()}"
+    config = """
+    terraform {
+      required_providers {
+        aws = {
+          source  = "hashicorp/aws"
+          version = ">= 6.23"
+        }
+      }
+    }
+
+    provider "aws" {
+      region                   = "us-east-1"
+      access_key               = "test"
+      secret_key               = "test"
+      skip_credentials_validation = true
+      skip_metadata_api_check   = true
+    }
+
+    resource "aws_s3_bucket" "example" {
+      name = "%s"
+    }
+    """ % bucket_name
+
+    with tempfile.TemporaryDirectory(delete=True) as temp_dir:
+        with open(os.path.join(temp_dir, "test.tf"), "w") as f:
+            f.write(config)
+
+        # we need the `terraform init` command to create a lock file, so it cannot be a `DRY_RUN`
+        run([TFLOCAL_BIN, "init"], cwd=temp_dir, env=dict(os.environ))
+        monkeypatch.setenv("DRY_RUN", "1")
+        run([TFLOCAL_BIN, "apply", "-auto-approve"], cwd=temp_dir, env=dict(os.environ))
+
+        override_file = os.path.join(temp_dir, "localstack_providers_override.tf")
+        assert check_override_file_exists(override_file)
+
+        with open(override_file, "r") as fp:
+            result = hcl2.load(fp)
+            endpoints = result["provider"][0]["aws"]["endpoints"][0]
+            assert "s3control" in endpoints
+            assert "mwaa" in endpoints
+
+            if aws_endpoint_url == "":
+                # we assert that if the `LOCALSTACK_HOSTNAME` isn't set, the default endpooint value is `localhost`
+                assert endpoints["sns"] == "http://localhost:4566"
+                # the MWAA endpoint needs to be "subdomain resolvable", so we override it with the default localhost
+                # localstack hostname
+                assert endpoints["mwaa"] == "http://mwaa.localhost.localstack.cloud:4566"
+                # s3 control will set the account id of the requested as a host prefix, so we also need a subdomain
+                # compatible host
+                assert endpoints["s3control"] == "http://localhost.localstack.cloud:4566"
+            else:
+                # if the user is manipulating the `LOCALSTACK_HOSTNAME`, they are responsible to make sure that the
+                # domain they set is subdomain compatible, so we respect their configuration
+                assert endpoints["sns"] == f"http://{endpoint_host}:4566"
+                assert endpoints["mwaa"] == f"http://mwaa.{endpoint_host}:4566"
+                assert endpoints["s3control"] == f"http://{endpoint_host}:4566"
+
+
 def test_dry_run(monkeypatch):
     monkeypatch.setenv("DRY_RUN", "1")
     state_bucket = "tf-state-dry-run"