Feature: Implement merge_headers

Implement and document `merge_headers`
for all other ESPs that can support it. (See #371
for base and Amazon SES implementation.)

Closes #374

Author: medmunds

CHANGELOG.rst

@@ -44,9 +44,12 @@ Breaking changes
 Features
 ~~~~~~~~
 
-* **Amazon SES:** Add new ``merge_headers`` option for per-recipient
-  headers with template sends. (Requires boto3 >= 1.34.98.)
-  (Thanks to `@carrerasrodrigo`_ the implementation.)
+* Add new ``merge_headers`` option for per-recipient headers with batch sends.
+  This can be helpful to send individual *List-Unsubscribe* headers (for example).
+  Supported for all current ESPs *except* MailerSend, Mandrill and Postal. See
+  `docs <https://anymail.dev/en/latest/sending/anymail_additions/#anymail.message.AnymailMessage.merge_headers>`__.
+  (Thanks to `@carrerasrodrigo`_ for the idea, and for the base and
+  Amazon SES implementations.)
 
 * **Amazon SES:** Allow extra headers, ``metadata``, ``merge_metadata``,
   and ``tags`` when sending with a ``template_id``.

anymail/backends/brevo.py

@@ -91,28 +91,32 @@ def init_payload(self):
         self.merge_data = {}
         self.metadata = {}
         self.merge_metadata = {}
+        self.merge_headers = {}
 
     def serialize_data(self):
         """Performs any necessary serialization on self.data, and returns the result."""
         if self.is_batch():
             # Burst data["to"] into data["messageVersions"]
             to_list = self.data.pop("to", [])
-            self.data["messageVersions"] = [
-                {"to": [to], "params": self.merge_data.get(to["email"])}
-                for to in to_list
-            ]
-            if self.merge_metadata:
-                # Merge global metadata with any per-recipient metadata.
-                # (Top-level X-Mailin-custom header is already set to global metadata,
-                # and will apply for recipients without a "headers" override.)
-                for version in self.data["messageVersions"]:
-                    to_email = version["to"][0]["email"]
-                    if to_email in self.merge_metadata:
-                        recipient_metadata = self.metadata.copy()
-                        recipient_metadata.update(self.merge_metadata[to_email])
-                        version["headers"] = {
-                            "X-Mailin-custom": self.serialize_json(recipient_metadata)
-                        }
+            self.data["messageVersions"] = []
+            for to in to_list:
+                to_email = to["email"]
+                version = {"to": [to]}
+                headers = CaseInsensitiveDict()
+                if to_email in self.merge_data:
+                    version["params"] = self.merge_data[to_email]
+                if to_email in self.merge_metadata:
+                    # Merge global metadata with any per-recipient metadata.
+                    # (Top-level X-Mailin-custom header already has global metadata,
+                    # and will apply for recipients without version headers.)
+                    recipient_metadata = self.metadata.copy()
+                    recipient_metadata.update(self.merge_metadata[to_email])
+                    headers["X-Mailin-custom"] = self.serialize_json(recipient_metadata)
+                if to_email in self.merge_headers:
+                    headers.update(self.merge_headers[to_email])
+                if headers:
+                    version["headers"] = headers
+                self.data["messageVersions"].append(version)
 
         if not self.data["headers"]:
             del self.data["headers"]  # don't send empty headers
@@ -212,6 +216,10 @@ def set_merge_metadata(self, merge_metadata):
         # Late-bound in serialize_data:
         self.merge_metadata = merge_metadata
 
+    def set_merge_headers(self, merge_headers):
+        # Late-bound in serialize_data:
+        self.merge_headers = merge_headers
+
     def set_send_at(self, send_at):
         try:
             start_time_iso = send_at.isoformat(timespec="milliseconds")

anymail/backends/mailgun.py

@@ -117,6 +117,7 @@ def __init__(self, message, defaults, backend, *args, **kwargs):
         self.merge_global_data = {}
         self.metadata = {}
         self.merge_metadata = {}
+        self.merge_headers = {}
         self.to_emails = []
 
         super().__init__(message, defaults, backend, auth=auth, *args, **kwargs)
@@ -191,6 +192,8 @@ def serialize_data(self):
     #     (E.g., Mailgun's custom-data "name" is set to "%recipient.name%", which picks
     #     up its per-recipient value from Mailgun's
     #     `recipient-variables[to_email]["name"]`.)
+    # (6) Anymail's `merge_headers` (per-recipient headers) maps to recipient-variables
+    #     prepended with 'h:'.
     #
     # If Anymail's `merge_data`, `template_id` (stored templates) and `metadata` (or
     # `merge_metadata`) are used together, there's a possibility of conflicting keys
@@ -268,6 +271,40 @@ def vkey(key):  # 'v:key'
                     {key: "%recipient.{}%".format(key) for key in merge_data_keys}
                 )
 
+        # (6) merge_headers --> Mailgun recipient_variables via 'h:'-prefixed keys
+        if self.merge_headers:
+
+            def hkey(field_name):  # 'h:Field-Name'
+                return "h:{}".format(field_name.title())
+
+            merge_header_fields = flatset(
+                recipient_headers.keys()
+                for recipient_headers in self.merge_headers.values()
+            )
+            merge_header_defaults = {
+                # existing h:Field-Name value (from extra_headers), or empty string
+                field: self.data.get(hkey(field), "")
+                for field in merge_header_fields
+            }
+            self.data.update(
+                # Set up 'h:Field-Name': '%recipient.h:Field-Name%' indirection
+                {
+                    hvar: f"%recipient.{hvar}%"
+                    for hvar in [hkey(field) for field in merge_header_fields]
+                }
+            )
+
+            for email in self.to_emails:
+                # Each recipient's recipient_variables needs _all_ merge header fields
+                recipient_headers = merge_header_defaults.copy()
+                recipient_headers.update(self.merge_headers.get(email, {}))
+                recipient_variables_for_headers = {
+                    hkey(field): value for field, value in recipient_headers.items()
+                }
+                recipient_variables.setdefault(email, {}).update(
+                    recipient_variables_for_headers
+                )
+
         # populate Mailgun params
         self.data.update({"v:%s" % key: value for key, value in custom_data.items()})
         if recipient_variables or self.is_batch():
@@ -308,8 +345,8 @@ def set_reply_to(self, emails):
             self.data["h:Reply-To"] = reply_to
 
     def set_extra_headers(self, headers):
-        for key, value in headers.items():
-            self.data["h:%s" % key] = value
+        for field, value in headers.items():
+            self.data["h:%s" % field.title()] = value
 
     def set_text_body(self, body):
         self.data["text"] = body
@@ -385,6 +422,9 @@ def set_merge_metadata(self, merge_metadata):
         # Processed at serialization time (to allow combining with merge_data)
         self.merge_metadata = merge_metadata
 
+    def set_merge_headers(self, merge_headers):
+        self.merge_headers = merge_headers
+
     def set_esp_extra(self, extra):
         self.data.update(extra)
         # Allow override of sender_domain via esp_extra

anymail/backends/mailjet.py

@@ -225,6 +225,13 @@ def set_merge_metadata(self, merge_metadata):
                     recipient_metadata = merge_metadata[email]
                 message["EventPayload"] = self.serialize_json(recipient_metadata)
 
+    def set_merge_headers(self, merge_headers):
+        self._burst_for_batch_send()
+        for message in self.data["Messages"]:
+            email = message["To"][0]["Email"]
+            if email in merge_headers:
+                message["Headers"] = merge_headers[email]
+
     def set_tags(self, tags):
         # The choices here are CustomID or Campaign, and Campaign seems closer
         # to how "tags" are handled by other ESPs -- e.g., you can view dashboard

anymail/backends/postmark.py

@@ -1,5 +1,7 @@
 import re
 
+from requests.structures import CaseInsensitiveDict
+
 from ..exceptions import AnymailRequestsAPIError
 from ..message import AnymailRecipientStatus
 from ..utils import (
@@ -209,6 +211,7 @@ def __init__(self, message, defaults, backend, *args, **kwargs):
         self.cc_and_bcc_emails = []  # needed for parse_recipient_status
         self.merge_data = None
         self.merge_metadata = None
+        self.merge_headers = {}
         super().__init__(message, defaults, backend, headers=headers, *args, **kwargs)
 
     def get_api_endpoint(self):
@@ -274,6 +277,18 @@ def data_for_recipient(self, to):
                 data["Metadata"].update(recipient_metadata)
             else:
                 data["Metadata"] = recipient_metadata
+        if to.addr_spec in self.merge_headers:
+            if "Headers" in data:
+                # merge global and recipient headers
+                headers = CaseInsensitiveDict(
+                    (item["Name"], item["Value"]) for item in data["Headers"]
+                )
+                headers.update(self.merge_headers[to.addr_spec])
+            else:
+                headers = self.merge_headers[to.addr_spec]
+            data["Headers"] = [
+                {"Name": name, "Value": value} for name, value in headers.items()
+            ]
         return data
 
     #
@@ -383,6 +398,10 @@ def set_merge_metadata(self, merge_metadata):
         # late-bind
         self.merge_metadata = merge_metadata
 
+    def set_merge_headers(self, merge_headers):
+        # late-bind
+        self.merge_headers = merge_headers
+
     def set_esp_extra(self, extra):
         self.data.update(extra)
         # Special handling for 'server_token':

anymail/backends/resend.py

@@ -98,6 +98,7 @@ def __init__(self, message, defaults, backend, *args, **kwargs):
         self.to_recipients = []  # for parse_recipient_status
         self.metadata = {}
         self.merge_metadata = {}
+        self.merge_headers = {}
         headers = kwargs.pop("headers", {})
         headers["Authorization"] = "Bearer %s" % backend.api_key
         headers["Content-Type"] = "application/json"
@@ -129,6 +130,14 @@ def serialize_data(self):
                     data["headers"]["X-Metadata"] = self.serialize_json(
                         recipient_metadata
                     )
+                if to.addr_spec in self.merge_headers:
+                    if "headers" in data:
+                        # Merge global headers (or X-Metadata from above)
+                        headers = CaseInsensitiveCasePreservingDict(data["headers"])
+                        headers.update(self.merge_headers[to.addr_spec])
+                    else:
+                        headers = self.merge_headers[to.addr_spec]
+                    data["headers"] = headers
                 payload.append(data)
 
         return self.serialize_json(payload)
@@ -284,5 +293,8 @@ def set_merge_data(self, merge_data):
     def set_merge_metadata(self, merge_metadata):
         self.merge_metadata = merge_metadata  # late bound in serialize_data
 
+    def set_merge_headers(self, merge_headers):
+        self.merge_headers = merge_headers  # late bound in serialize_data
+
     def set_esp_extra(self, extra):
         self.data.update(extra)

anymail/backends/sendgrid.py

@@ -92,6 +92,7 @@ def __init__(self, message, defaults, backend, *args, **kwargs):
         self.merge_data = {}  # late-bound per-recipient data
         self.merge_global_data = {}
         self.merge_metadata = {}
+        self.merge_headers = {}
 
         http_headers = kwargs.pop("headers", {})
         http_headers["Authorization"] = "Bearer %s" % backend.api_key
@@ -116,6 +117,7 @@ def serialize_data(self):
             self.expand_personalizations_for_batch()
         self.build_merge_data()
         self.build_merge_metadata()
+        self.build_merge_headers()
         if self.generate_message_id:
             self.set_anymail_id()
 
@@ -216,6 +218,15 @@ def build_merge_metadata(self):
                     recipient_custom_args = self.transform_metadata(recipient_metadata)
                     personalization["custom_args"] = recipient_custom_args
 
+    def build_merge_headers(self):
+        if self.merge_headers:
+            for personalization in self.data["personalizations"]:
+                assert len(personalization["to"]) == 1
+                recipient_email = personalization["to"][0]["email"]
+                recipient_headers = self.merge_headers.get(recipient_email)
+                if recipient_headers:
+                    personalization["headers"] = recipient_headers
+
     #
     # Payload construction
     #
@@ -374,6 +385,11 @@ def set_merge_metadata(self, merge_metadata):
         # and merge_field_format.
         self.merge_metadata = merge_metadata
 
+    def set_merge_headers(self, merge_headers):
+        # Becomes personalizations[...]['headers'] in
+        # build_merge_data
+        self.merge_headers = merge_headers
+
     def set_esp_extra(self, extra):
         self.merge_field_format = extra.pop(
             "merge_field_format", self.merge_field_format

anymail/backends/sparkpost.py

@@ -242,6 +242,36 @@ def set_merge_metadata(self, merge_metadata):
             if to_email in merge_metadata:
                 recipient["metadata"] = merge_metadata[to_email]
 
+    def set_merge_headers(self, merge_headers):
+        def header_var(field):
+            return "Header__" + field.title().replace("-", "_")
+
+        merge_header_fields = set()
+
+        for recipient in self.data["recipients"]:
+            to_email = recipient["address"]["email"]
+            if to_email in merge_headers:
+                recipient_headers = merge_headers[to_email]
+                recipient.setdefault("substitution_data", {}).update(
+                    {header_var(key): value for key, value in recipient_headers.items()}
+                )
+                merge_header_fields.update(recipient_headers.keys())
+
+        if merge_header_fields:
+            headers = self.data.setdefault("content", {}).setdefault("headers", {})
+            # Global substitution_data supplies defaults for defined headers:
+            self.data.setdefault("substitution_data", {}).update(
+                {
+                    header_var(field): headers[field]
+                    for field in merge_header_fields
+                    if field in headers
+                }
+            )
+            # Indirect merge_headers through substitution_data:
+            headers.update(
+                {field: "{{%s}}" % header_var(field) for field in merge_header_fields}
+            )
+
     def set_send_at(self, send_at):
         try:
             start_time = send_at.replace(microsecond=0).isoformat()

anymail/backends/unisender_go.py

@@ -161,7 +161,7 @@ def serialize_data(self) -> str:
             headers.pop("to", None)
             if headers.pop("cc", None):
                 self.unsupported_feature(
-                    "cc with batch send (merge_data or merge_metadata)"
+                    "cc with batch send (merge_data, merge_metadata, or merge_headers)"
                 )
 
         if not headers:
@@ -339,5 +339,26 @@ def set_merge_metadata(self, merge_metadata: dict[str, str]) -> None:
             if recipient_email in merge_metadata:
                 recipient["metadata"] = merge_metadata[recipient_email]
 
+    # Unisender Go supports header substitution only with List-Unsubscribe.
+    # (See https://godocs.unisender.ru/web-api-ref#email-send under "substitutions".)
+    SUPPORTED_MERGE_HEADERS = {"List-Unsubscribe"}
+
+    def set_merge_headers(self, merge_headers: dict[str, dict[str, str]]) -> None:
+        assert self.data["recipients"]  # must be called after set_to
+        if merge_headers:
+            for recipient in self.data["recipients"]:
+                recipient_email = recipient["email"]
+                for key, value in merge_headers.get(recipient_email, {}).items():
+                    field = key.title()  # canonicalize field name capitalization
+                    if field in self.SUPPORTED_MERGE_HEADERS:
+                        # Set up a substitution for Header__Field_Name
+                        field_sub = "Header__" + field.replace("-", "_")
+                        recipient.setdefault("substitutions", {})[field_sub] = value
+                        self.data.setdefault("headers", {})[field] = (
+                            "{{%s}}" % field_sub
+                        )
+                    else:
+                        self.unsupported_feature(f"{field!r} in merge_headers")
+
     def set_esp_extra(self, extra: dict) -> None:
         update_deep(self.data, extra)

docs/esps/amazon_ses.rst

@@ -96,6 +96,14 @@ Limitations and quirks
 **No delayed sending**
   Amazon SES does not support :attr:`~anymail.message.AnymailMessage.send_at`.
 
+**Merge features require template_id**
+  Anymail's :attr:`~anymail.message.AnymailMessage.merge_headers`,
+  :attr:`~anymail.message.AnymailMessage.merge_metadata`,
+  :attr:`~anymail.message.AnymailMessage.merge_data`, and
+  :attr:`~anymail.message.AnymailMessage.merge_global_data` are only supported
+  when sending :ref:`templated messages <amazon-ses-templates>`
+  (using Anymail's :attr:`~anymail.message.AnymailMessage.template_id`).
+
 **No global send defaults for non-Anymail options**
   With the Amazon SES backend, Anymail's :ref:`global send defaults <send-defaults>`
   are only supported for Anymail's added message options (like