Fix versioned service endpoints (#81)

Author: bentsku

README.md

@@ -77,6 +77,7 @@ ADDITIONAL_TF_OVERRIDE_LOCATIONS=/path/to/module1,path/to/module2 tflocal plan
 
 ## Change Log
 
+* v0.23.1: Fix endpoint overrides for Terraform AWS provider >= 6.0.0-beta2
 * v0.23.0: Add support for `terraform_remote_state` with `s3` backend to read the state stored in local S3 backend; fix S3 backend config detection with multiple Terraform blocks
 * v0.22.0: Fix S3 backend forcing DynamoDB State Lock to be enabled by default
 * v0.21.0: Add ability to drop an override file in additional locations

bin/tflocal

@@ -56,6 +56,8 @@ LOCALSTACK_HOSTNAME = (
     or "localhost"
 )
 EDGE_PORT = int(urlparse(AWS_ENDPOINT_URL).port or os.environ.get("EDGE_PORT") or 4566)
+AWS_PROVIDER_NAME_SUFFIX = "/hashicorp/aws"
+AWS_PROVIDER_VERSION: Optional[version.Version] = None
 TF_VERSION: Optional[version.Version] = None
 TF_PROVIDER_CONFIG = """
 provider "aws" {
@@ -128,6 +130,15 @@ SERVICE_ALIASES = [
 ]
 # service names to be excluded (not yet available in TF)
 SERVICE_EXCLUSIONS = ["meteringmarketplace"]
+
+# we can exclude some service endpoints based on the AWS provider version
+# those limits are exclusive, meaning 6.0.0b2 is the first version to fail with those endpoints, so only a lower version
+# will have that setting
+VERSIONED_SERVICE_EXCLUSIONS = {
+    "iotanalytics": {"min": version.Version("0"), "max": version.Version("6.0.0b2")},
+    "iotevents": {"min": version.Version("0"), "max": version.Version("6.0.0b2")},
+}
+
 # maps services to be replaced with alternative names
 # skip services which do not have equivalent endpoint overrides
 # see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/guides/custom-service-endpoints
@@ -173,7 +184,8 @@ def create_provider_config_file(provider_file_path: str, provider_aliases=None)
 
     # create list of service names
     services = list(config.get_service_ports())
-    services = [srvc for srvc in services if srvc not in SERVICE_EXCLUSIONS]
+    services = [srvc for srvc in services if srvc not in SERVICE_EXCLUSIONS and is_service_endpoint_supported(srvc)]
+
     services = [s.replace("-", "") for s in services]
     for old, new in SERVICE_REPLACEMENTS.items():
         try:
@@ -606,6 +618,33 @@ def get_tf_version(env):
     TF_VERSION = version.parse(json.loads(output)["terraform_version"])
 
 
+def get_provider_version_from_lock_file() -> Optional[version.Version]:
+    global AWS_PROVIDER_VERSION
+    lock_file = os.path.join(get_default_provider_folder_path(), ".terraform.lock.hcl")
+
+    if not os.path.exists(lock_file):
+        return
+
+    provider_version = None
+    with open(lock_file, "r") as fp:
+        result = hcl2.load(fp)
+        for provider in result.get("provider", []):
+            for provider_name, provider_config in provider.items():
+                if provider_name.endswith(AWS_PROVIDER_NAME_SUFFIX):
+                    provider_version = provider_config.get("version")
+
+    if provider_version:
+        AWS_PROVIDER_VERSION = version.parse(provider_version)
+
+
+def is_service_endpoint_supported(service_name: str) -> bool:
+    if service_name not in VERSIONED_SERVICE_EXCLUSIONS or not AWS_PROVIDER_VERSION:
+        return True
+
+    supported_versions = VERSIONED_SERVICE_EXCLUSIONS[service_name]
+    return supported_versions["min"] < AWS_PROVIDER_VERSION < supported_versions["max"]
+
+
 def run_tf_exec(cmd, env):
     """Run terraform using os.exec - can be useful as it does not require any I/O
     handling for stdin/out/err. Does *not* allow us to perform any cleanup logic."""
@@ -686,6 +725,9 @@ def main():
         print(f"Unable to determine version. See error message for details: {e}")
         exit(1)
 
+    if len(sys.argv) > 1 and sys.argv[1] != "init":
+        get_provider_version_from_lock_file()
+
     config_override_files = []
 
     for folder_path in get_folder_paths_that_require_an_override_file():

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = terraform-local
-version = 0.23.0
+version = 0.23.1
 url = https://github.com/localstack/terraform-local
 author = LocalStack Team
 author_email = info@localstack.cloud

tests/test_apply.py

@@ -390,6 +390,60 @@ def test_s3_remote_data_source_with_workspace(monkeypatch):
         assert result["data"][1]["terraform_remote_state"]["build_infra"]["workspace"] == "build"
 
 
+@pytest.mark.parametrize("provider_version", ["5.99.1", "6.0.0-beta2"])
+def test_versioned_endpoints(monkeypatch, provider_version):
+    sns_topic_name = f"test-topic-{short_uid()}"
+    config = """
+    terraform {
+      required_providers {
+        aws = {
+          source  = "hashicorp/aws"
+          version = "= %s"
+        }
+      }
+    }
+
+    provider "aws" {
+      region                   = "us-east-1"
+      access_key               = "test"
+      secret_key               = "test"
+      skip_credentials_validation = true
+      skip_metadata_api_check   = true
+      skip_requesting_account_id = true
+      endpoints {
+        sns = "http://localhost:4566"
+      }
+    }
+
+    resource "aws_sns_topic" "example" {
+      name = "%s"
+    }
+    """ % (provider_version, sns_topic_name)
+
+    with tempfile.TemporaryDirectory(delete=True) as temp_dir:
+        with open(os.path.join(temp_dir, "test.tf"), "w") as f:
+            f.write(config)
+
+        # we need the `terraform init` command to create a lock file, so it cannot be a `DRY_RUN`
+        run([TFLOCAL_BIN, "init"], cwd=temp_dir, env=dict(os.environ))
+        monkeypatch.setenv("DRY_RUN", "1")
+        run([TFLOCAL_BIN, "apply", "-auto-approve"], cwd=temp_dir, env=dict(os.environ))
+
+        override_file = os.path.join(temp_dir, "localstack_providers_override.tf")
+        assert check_override_file_exists(override_file)
+
+        with open(override_file, "r") as fp:
+            result = hcl2.load(fp)
+            endpoints = result["provider"][0]["aws"]["endpoints"][0]
+            if provider_version == "5.99.1":
+                assert "iotanalytics" in endpoints
+                assert "iotevents" in endpoints
+            else:
+                # we add this assertion to be sure, but Terraform wouldn't deploy with them
+                assert "iotanalytics" not in endpoints
+                assert "iotevents" not in endpoints
+
+
 def test_dry_run(monkeypatch):
     monkeypatch.setenv("DRY_RUN", "1")
     state_bucket = "tf-state-dry-run"
@@ -638,8 +692,14 @@ def get_version():
     return version.parse(json.loads(output)["terraform_version"])
 
 
-def deploy_tf_script(script: str, cleanup: bool = True, env_vars: Dict[str, str] = None, user_input: str = None):
-    # TODO the delete keyword was added in python 3.12, and the README and setup.cfg claims compatibility with earlier python versions
+def deploy_tf_script(
+    script: str,
+    cleanup: bool = True,
+    env_vars: Dict[str, str] = None,
+    user_input: str = None,
+):
+    # TODO the delete keyword was added in python 3.12, and the README and setup.cfg claims compatibility
+    #  with earlier python versions
     with tempfile.TemporaryDirectory(delete=cleanup) as temp_dir:
         with open(os.path.join(temp_dir, "test.tf"), "w") as f:
             f.write(script)