gh-130453: pygettext: Allow specifying multiple keywords with the same function name

Lib/test/test_tools/i18n_data/multiple_keywords.pot

@@ -0,0 +1,38 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) YEAR ORGANIZATION
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\n"
+"POT-Creation-Date: 2000-01-01 00:00+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+"Generated-By: pygettext.py 1.5\n"
+
+
+#: multiple_keywords.py:3
+msgid "bar"
+msgstr ""
+
+#: multiple_keywords.py:5
+msgctxt "baz"
+msgid "qux"
+msgstr ""
+
+#: multiple_keywords.py:9
+msgctxt "corge"
+msgid "grault"
+msgstr ""
+
+#: multiple_keywords.py:11
+msgctxt "xyzzy"
+msgid "foo"
+msgid_plural "foos"
+msgstr[0] ""
+msgstr[1] ""
+

Lib/test/test_tools/i18n_data/multiple_keywords.py

@@ -0,0 +1,11 @@
+from gettext import gettext as foo
+
+foo('bar')
+
+foo('baz', 'qux')
+
+# The 't' specifier is not supported, so the following
+# call is extracted as pgettext instead of ngettext.
+foo('corge', 'grault', 1)
+
+foo('xyzzy', 'foo', 'foos', 1)

Lib/test/test_tools/test_i18n.py

@@ -18,7 +18,7 @@
 
 
 with imports_under_tool("i18n"):
-    from pygettext import parse_spec
+    from pygettext import parse_spec, process_keywords, DEFAULTKEYWORDS
 
 
 def normalize_POT_file(pot):
@@ -483,16 +483,16 @@ def test_comments_not_extracted_without_tags(self):
 
     def test_parse_keyword_spec(self):
         valid = (
-            ('foo', ('foo', {0: 'msgid'})),
-            ('foo:1', ('foo', {0: 'msgid'})),
-            ('foo:1,2', ('foo', {0: 'msgid', 1: 'msgid_plural'})),
-            ('foo:1, 2', ('foo', {0: 'msgid', 1: 'msgid_plural'})),
-            ('foo:1,2c', ('foo', {0: 'msgid', 1: 'msgctxt'})),
-            ('foo:2c,1', ('foo', {0: 'msgid', 1: 'msgctxt'})),
-            ('foo:2c ,1', ('foo', {0: 'msgid', 1: 'msgctxt'})),
-            ('foo:1,2,3c', ('foo', {0: 'msgid', 1: 'msgid_plural', 2: 'msgctxt'})),
-            ('foo:1, 2, 3c', ('foo', {0: 'msgid', 1: 'msgid_plural', 2: 'msgctxt'})),
-            ('foo:3c,1,2', ('foo', {0: 'msgid', 1: 'msgid_plural', 2: 'msgctxt'})),
+            ('foo', ('foo', {'msgid': 0})),
+            ('foo:1', ('foo', {'msgid': 0})),
+            ('foo:1,2', ('foo', {'msgid': 0, 'msgid_plural': 1})),
+            ('foo:1, 2', ('foo', {'msgid': 0, 'msgid_plural': 1})),
+            ('foo:1,2c', ('foo', {'msgid': 0, 'msgctxt': 1})),
+            ('foo:2c,1', ('foo', {'msgid': 0, 'msgctxt': 1})),
+            ('foo:2c ,1', ('foo', {'msgid': 0, 'msgctxt': 1})),
+            ('foo:1,2,3c', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})),
+            ('foo:1, 2, 3c', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})),
+            ('foo:3c,1,2', ('foo', {'msgid': 0, 'msgid_plural': 1, 'msgctxt': 2})),
         )
         for spec, expected in valid:
             with self.subTest(spec=spec):
@@ -516,6 +516,59 @@ def test_parse_keyword_spec(self):
                     parse_spec(spec)
                 self.assertEqual(str(cm.exception), message)
 
+    def test_process_keywords(self):
+        default_keywords = {name: [spec] for name, spec
+                            in DEFAULTKEYWORDS.items()}
+        inputs = (
+            (['foo'], True),
+            (['_:1,2'], True),
+            (['foo', 'foo:1,2'], True),
+            (['foo'], False),
+            (['_:1,2', '_:1c,2,3', 'pgettext'], False),
+        )
+        expected = (
+            {'foo': [{'msgid': 0}]},
+            {'_': [{'msgid': 0, 'msgid_plural': 1}]},
+            {'foo': [{'msgid': 0}, {'msgid': 0, 'msgid_plural': 1}]},
+            default_keywords | {'foo': [{'msgid': 0}]},
+            default_keywords | {'_': [{'msgid': 0, 'msgid_plural': 1},
+                                      {'msgctxt': 0, 'msgid': 1, 'msgid_plural': 2}],
+                                'pgettext': [{'msgid': 0}]},
+        )
+        for (keywords, no_default_keywords), expected in zip(inputs, expected):
+            with self.subTest(keywords=keywords,
+                              no_default_keywords=no_default_keywords):
+                processed = process_keywords(
+                    keywords,
+                    no_default_keywords=no_default_keywords)
+                self.assertEqual(processed, expected)
+
+    def test_multiple_keywords_same_funcname_errors(self):
+        # If at least one keyword spec for a given funcname matches,
+        # no error should be printed.
+        msgids, stderr = self.extract_from_str(dedent('''\
+        _("foo", 42)
+        _(42, "bar")
+        '''), args=('--keyword=_:1', '--keyword=_:2'), with_stderr=True)
+        self.assertIn('foo', msgids)
+        self.assertIn('bar', msgids)
+        self.assertEqual(stderr, b'')
+
+        # If no keyword spec for a given funcname matches,
+        # all errors are printed.
+        msgids, stderr = self.extract_from_str(dedent('''\
+        _(x, 42)
+        _(42, y)
+        '''), args=('--keyword=_:1', '--keyword=_:2'), with_stderr=True,
+              strict=False)
+        self.assertEqual(msgids, [''])
+        self.assertEqual(
+            stderr,
+            b'*** test.py:1: Expected a string constant for argument 1, got x\n'
+            b'*** test.py:1: Expected a string constant for argument 2, got 42\n'
+            b'*** test.py:2: Expected a string constant for argument 1, got 42\n'
+            b'*** test.py:2: Expected a string constant for argument 2, got y\n')
+
 
 def extract_from_snapshots():
     snapshots = {
@@ -526,6 +579,10 @@ def extract_from_snapshots():
         'custom_keywords.py': ('--keyword=foo', '--keyword=nfoo:1,2',
                                '--keyword=pfoo:1c,2',
                                '--keyword=npfoo:1c,2,3', '--keyword=_:1,2'),
+        'multiple_keywords.py': ('--keyword=foo:1c,2,3', '--keyword=foo:1c,2',
+                                 '--keyword=foo:1,2',
+                                 # repeat a keyword to make sure it is extracted only once
+                                 '--keyword=foo', '--keyword=foo'),
     }
 
     for filename, args in snapshots.items():

Misc/NEWS.d/next/Tools-Demos/2025-03-10-08-19-22.gh-issue-130453.9B0x8k.rst

@@ -0,0 +1,2 @@
+Allow passing multiple keyword arguments with the same function name in
+:program:`pygettext`.

Tools/i18n/pygettext.py

@@ -282,15 +282,15 @@ def getFilesForName(name):
 # Key is the function name, value is a dictionary mapping argument positions to the
 # type of the argument. The type is one of 'msgid', 'msgid_plural', or 'msgctxt'.
 DEFAULTKEYWORDS = {
-    '_': {0: 'msgid'},
-    'gettext': {0: 'msgid'},
-    'ngettext': {0: 'msgid', 1: 'msgid_plural'},
-    'pgettext': {0: 'msgctxt', 1: 'msgid'},
-    'npgettext': {0: 'msgctxt', 1: 'msgid', 2: 'msgid_plural'},
-    'dgettext': {1: 'msgid'},
-    'dngettext': {1: 'msgid', 2: 'msgid_plural'},
-    'dpgettext': {1: 'msgctxt', 2: 'msgid'},
-    'dnpgettext': {1: 'msgctxt', 2: 'msgid', 3: 'msgid_plural'},
+    '_': {'msgid': 0},
+    'gettext': {'msgid': 0},
+    'ngettext': {'msgid': 0, 'msgid_plural': 1},
+    'pgettext': {'msgctxt': 0, 'msgid': 1},
+    'npgettext': {'msgctxt': 0, 'msgid': 1, 'msgid_plural': 2},
+    'dgettext': {'msgid': 1},
+    'dngettext': {'msgid': 1, 'msgid_plural': 2},
+    'dpgettext': {'msgctxt': 1, 'msgid': 2},
+    'dnpgettext': {'msgctxt': 1, 'msgid': 2, 'msgid_plural': 3},
 }
 
 
@@ -327,7 +327,7 @@ def parse_spec(spec):
     parts = spec.strip().split(':', 1)
     if len(parts) == 1:
         name = parts[0]
-        return name, {0: 'msgid'}
+        return name, {'msgid': 0}
 
     name, args = parts
     if not args:
@@ -373,7 +373,23 @@ def parse_spec(spec):
         raise ValueError(f'Invalid keyword spec {spec!r}: '
                          'msgctxt cannot appear without msgid')
 
-    return name, {v: k for k, v in result.items()}
+    return name, result
+
+
+def process_keywords(keywords, *, no_default_keywords):
+    custom_keywords_list = [parse_spec(spec) for spec in keywords]
+    custom_keywords = {}
+    for name, spec in custom_keywords_list:
+        if name not in custom_keywords:
+            custom_keywords[name] = []
+        custom_keywords[name].append(spec)
+
+    if no_default_keywords:
+        return custom_keywords
+
+    default_keywords = {name: [spec] for name, spec in DEFAULTKEYWORDS.items()}
+    # custom keywords override default keywords
+    return default_keywords | custom_keywords
 
 
 @dataclass(frozen=True)
@@ -459,32 +475,41 @@ def _extract_docstring(self, node):
 
     def _extract_message(self, node):
         func_name = self._get_func_name(node)
-        spec = self.options.keywords.get(func_name)
-        if spec is None:
-            return
+        errors = []
+        for spec in self.options.keywords.get(func_name, []):
+            err = self._extract_message_with_spec(node, spec)
+            if err is None:
+                break
+            errors.append(err)
+        else:
+            for err in errors:
+                print(err, file=sys.stderr)
+
+    def _extract_message_with_spec(self, node, spec):
+        """Extract a gettext call with the given spec.
 
-        max_index = max(spec)
+        Return `None` if the gettext call was successfully extracted,
+        otherwise return an error message.
+        """
+        max_index = max(spec.values())
         has_var_positional = any(isinstance(arg, ast.Starred) for
                                  arg in node.args[:max_index+1])
         if has_var_positional:
-            print(f'*** {self.filename}:{node.lineno}: Variable positional '
-                  f'arguments are not allowed in gettext calls', file=sys.stderr)
-            return
+            return (f'*** {self.filename}:{node.lineno}: Variable positional '
+                   f'arguments are not allowed in gettext calls')
 
         if max_index >= len(node.args):
-            print(f'*** {self.filename}:{node.lineno}: Expected at least '
-                  f'{max(spec) + 1} positional argument(s) in gettext call, '
-                  f'got {len(node.args)}', file=sys.stderr)
-            return
+            return (f'*** {self.filename}:{node.lineno}: Expected at least '
+                    f'{max_index + 1} positional argument(s) in gettext call, '
+                    f'got {len(node.args)}')
 
         msg_data = {}
-        for position, arg_type in spec.items():
+        for arg_type, position in spec.items():
             arg = node.args[position]
             if not self._is_string_const(arg):
-                print(f'*** {self.filename}:{arg.lineno}: Expected a string '
-                      f'constant for argument {position + 1}, '
-                      f'got {ast.unparse(arg)}', file=sys.stderr)
-                return
+                return (f'*** {self.filename}:{arg.lineno}: Expected a string '
+                        f'constant for argument {position + 1}, '
+                        f'got {ast.unparse(arg)}')
             msg_data[arg_type] = arg.value
 
         lineno = node.lineno
@@ -729,15 +754,12 @@ class Options:
 
     # calculate all keywords
     try:
-        custom_keywords = dict(parse_spec(spec) for spec in options.keywords)
+        options.keywords = process_keywords(
+            options.keywords,
+            no_default_keywords=no_default_keywords)
     except ValueError as e:
         print(e, file=sys.stderr)
         sys.exit(1)
-    options.keywords = {}
-    if not no_default_keywords:
-        options.keywords |= DEFAULTKEYWORDS
-    # custom keywords override default keywords
-    options.keywords |= custom_keywords
 
     # initialize list of strings to exclude
     if options.excludefilename: