Amazon SES: support headers with template

Use new SES v2 SendBulkEmail ReplacementHeaders param
to support features that require custom headers,
including `extra_headers`, `metadata`,
`merge_metadata` and `tags`.

Update integration tests and docs

Closes #375

Author: medmunds

CHANGELOG.rst

@@ -48,6 +48,10 @@ Features
   headers with template sends. (Requires boto3 >= 1.34.98.)
   (Thanks to `@carrerasrodrigo`_ the implementation.)
 
+* **Amazon SES:** Allow extra headers, ``metadata``, ``merge_metadata``,
+  and ``tags`` when sending with a ``template_id``.
+  (Requires boto3 v1.34.98 or later.)
+
 
 v10.3
 -----

anymail/backends/amazon_ses.py

@@ -2,6 +2,8 @@
 import email.encoders
 import email.policy
 
+from requests.structures import CaseInsensitiveDict
+
 from .. import __version__ as ANYMAIL_VERSION
 from ..exceptions import AnymailAPIError, AnymailImproperlyInstalled
 from ..message import AnymailRecipientStatus
@@ -339,10 +341,14 @@ class AmazonSESV2SendBulkEmailPayload(AmazonSESBasePayload):
 
     def init_payload(self):
         super().init_payload()
-        # late-bind recipients and merge_data in finalize_payload
+        # late-bind in finalize_payload:
         self.recipients = {"to": [], "cc": [], "bcc": []}
         self.merge_data = {}
+        self.headers = {}
         self.merge_headers = {}
+        self.metadata = {}
+        self.merge_metadata = {}
+        self.tags = []
 
     def finalize_payload(self):
         # Build BulkEmailEntries from recipients and merge_data.
@@ -372,11 +378,26 @@ def finalize_payload(self):
                 },
             }
 
-            if len(self.merge_headers) > 0:
-                entry["ReplacementHeaders"] = [
-                    {"Name": key, "Value": value}
-                    for key, value in self.merge_headers.get(to.addr_spec, {}).items()
+            replacement_headers = []
+            if self.headers or to.addr_spec in self.merge_headers:
+                headers = CaseInsensitiveDict(self.headers)
+                headers.update(self.merge_headers.get(to.addr_spec, {}))
+                replacement_headers += [
+                    {"Name": key, "Value": value} for key, value in headers.items()
+                ]
+            if self.metadata or to.addr_spec in self.merge_metadata:
+                metadata = self.metadata.copy()
+                metadata.update(self.merge_metadata.get(to.addr_spec, {}))
+                if metadata:
+                    replacement_headers.append(
+                        {"Name": "X-Metadata", "Value": self.serialize_json(metadata)}
+                    )
+            if self.tags:
+                replacement_headers += [
+                    {"Name": "X-Tag", "Value": tag} for tag in self.tags
                 ]
+            if replacement_headers:
+                entry["ReplacementHeaders"] = replacement_headers
             self.params["BulkEmailEntries"].append(entry)
 
     def parse_recipient_status(self, response):
@@ -446,7 +467,7 @@ def set_reply_to(self, emails):
             self.params["ReplyToAddresses"] = [email.address for email in emails]
 
     def set_extra_headers(self, headers):
-        self.unsupported_feature("extra_headers with template")
+        self.headers = headers
 
     def set_text_body(self, body):
         if body:
@@ -468,27 +489,26 @@ def set_envelope_sender(self, email):
         self.params["FeedbackForwardingEmailAddress"] = email.addr_spec
 
     def set_metadata(self, metadata):
-        # no custom headers with SendBulkEmail
-        self.unsupported_feature("metadata with template")
+        self.metadata = metadata
+
+    def set_merge_metadata(self, merge_metadata):
+        self.merge_metadata = merge_metadata
 
     def set_tags(self, tags):
-        # no custom headers with SendBulkEmail, but support
-        # AMAZON_SES_MESSAGE_TAG_NAME if used (see tags/metadata in
-        # AmazonSESV2SendEmailPayload for more info)
-        if tags:
-            if self.backend.message_tag_name is not None:
-                if len(tags) > 1:
-                    self.unsupported_feature(
-                        "multiple tags with the AMAZON_SES_MESSAGE_TAG_NAME setting"
-                    )
-                self.params["DefaultEmailTags"] = [
-                    {"Name": self.backend.message_tag_name, "Value": tags[0]}
-                ]
-            else:
+        self.tags = tags
+
+        # Also *optionally* pass a single Message Tag if the AMAZON_SES_MESSAGE_TAG_NAME
+        # Anymail setting is set (default no). The AWS API restricts tag content in this
+        # case. (This is useful for dashboard segmentation; use esp_extra["Tags"] for
+        # anything more complex.)
+        if tags and self.backend.message_tag_name is not None:
+            if len(tags) > 1:
                 self.unsupported_feature(
-                    "tags with template (unless using the"
-                    " AMAZON_SES_MESSAGE_TAG_NAME setting)"
+                    "multiple tags with the AMAZON_SES_MESSAGE_TAG_NAME setting"
                 )
+            self.params["DefaultEmailTags"] = [
+                {"Name": self.backend.message_tag_name, "Value": tags[0]}
+            ]
 
     def set_template_id(self, template_id):
         # DefaultContent.Template.TemplateName

docs/esps/amazon_ses.rst

@@ -68,6 +68,11 @@ setting to customize the Boto session.
 Limitations and quirks
 ----------------------
 
+.. versionchanged:: 11.0
+
+    Anymail's :attr:`~anymail.message.AnymailMessage.merge_metadata`
+    is now supported.
+
 **Hard throttling**
   Like most ESPs, Amazon SES `throttles sending`_ for new customers. But unlike
   most ESPs, SES does not queue and slowly release throttled messages. Instead, it
@@ -80,11 +85,6 @@ Limitations and quirks
   :attr:`~anymail.message.AnymailMessage.tags` feature. See :ref:`amazon-ses-tags`
   below for more information and additional options.
 
-**No merge_metadata**
-  Amazon SES's batch sending API does not support the custom headers Anymail uses
-  for metadata, so Anymail's :attr:`~anymail.message.AnymailMessage.merge_metadata`
-  feature is not available. (See :ref:`amazon-ses-tags` below for more information.)
-
 **Open and click tracking overrides**
   Anymail's :attr:`~anymail.message.AnymailMessage.track_opens` and
   :attr:`~anymail.message.AnymailMessage.track_clicks` are not supported.
@@ -126,7 +126,7 @@ Limitations and quirks
   signal, and using it will likely prevent delivery of your email.)
 
 **Template limitations**
-  Messages sent with templates have a number of additional limitations, such as not
+  Messages sent with templates have some additional limitations, such as not
   supporting attachments. See :ref:`amazon-ses-templates` below.
 
 
@@ -195,12 +195,7 @@ characters.
 
 For more complex use cases, set the SES ``EmailTags`` parameter (or ``DefaultEmailTags``
 for template sends) directly in Anymail's :ref:`esp_extra <amazon-ses-esp-extra>`. See
-the example below. (Because custom headers do not work with SES's SendBulkEmail call,
-esp_extra ``DefaultEmailTags`` is the only way to attach data to SES messages also using
-Anymail's :attr:`~anymail.message.AnymailMessage.template_id` and
-:attr:`~anymail.message.AnymailMessage.merge_data` features, and
-:attr:`~anymail.message.AnymailMessage.merge_metadata` cannot be supported.)
-
+the example below.
 
 .. _Introducing Sending Metrics:
     https://aws.amazon.com/blogs/ses/introducing-sending-metrics/
@@ -264,9 +259,10 @@ See Amazon's `Sending personalized email`_ guide for more information.
 When you set a message's :attr:`~anymail.message.AnymailMessage.template_id`
 to the name of one of your SES templates, Anymail will use the SES v2 `SendBulkEmail`_
 call to send template messages personalized with data
-from Anymail's normalized :attr:`~anymail.message.AnymailMessage.merge_data`
-and :attr:`~anymail.message.AnymailMessage.merge_global_data`
-message attributes.
+from Anymail's normalized :attr:`~anymail.message.AnymailMessage.merge_data`,
+:attr:`~anymail.message.AnymailMessage.merge_global_data`,
+:attr:`~anymail.message.AnymailMessage.merge_metadata`, and
+:attr:`~anymail.message.AnymailMessage.merge_headers` message attributes.
 
   .. code-block:: python
 
@@ -284,17 +280,21 @@ message attributes.
           'ship_date': "May 15",
       }
 
-Amazon's templated email APIs don't support several features available for regular email.
+Amazon's templated email APIs don't support a few features available for regular email.
 When :attr:`~anymail.message.AnymailMessage.template_id` is used:
 
-* Attachments and alternative parts (including AMPHTML) are not supported
-* Extra headers are not supported
+* Attachments and inline images are not supported
+* Alternative parts (including AMPHTML) are not supported
 * Overriding the template's subject or body is not supported
-* Anymail's :attr:`~anymail.message.AnymailMessage.metadata` is not supported
-* Anymail's :attr:`~anymail.message.AnymailMessage.tags` are only supported
-  with the :setting:`AMAZON_SES_MESSAGE_TAG_NAME <ANYMAIL_AMAZON_SES_MESSAGE_TAG_NAME>`
-  setting; only a single tag is allowed, and the tag is not directly available
-  to webhooks. (See :ref:`amazon-ses-tags` above.)
+
+.. versionchanged:: 11.0
+
+    Extra headers, :attr:`~anymail.message.AnymailMessage.metadata`,
+    :attr:`~anymail.message.AnymailMessage.merge_metadata`, and
+    :attr:`~anymail.message.AnymailMessage.tags` are now fully supported
+    when using :attr:`~anymail.message.AnymailMessage.template_id`.
+    (This requires :pypi:`boto3` v1.34.98 or later, which enables the
+    ReplacementHeaders parameter for SendBulkEmail.)
 
 .. _Sending personalized email:
    https://docs.aws.amazon.com/ses/latest/DeveloperGuide/send-personalized-email-api.html