From dc136768180506c5823c5ecc1dbaabee81f77fd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Simon?= Date: Sat, 30 Aug 2025 18:42:11 +0200 Subject: [PATCH 1/6] PyREPL module completion: hardcode special stdlib submodules --- Lib/_pyrepl/_module_completer.py | 19 ++++++++++++-- Lib/test/test_pyrepl/test_pyrepl.py | 40 +++++++++++++++++++++-------- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index 1e9462a42156d4..5a2037a66b81fe 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -16,6 +16,15 @@ from typing import Any, Iterable, Iterator, Mapping +HARDCODED_SUBMODULES = { + # Standard library submodules that are not detected by pkgutil.iter_modules + # but can be imported, so should be proposed in completion + "collections": ["abc"], + "os": ["path"], + "xml.parsers.expat": ["errors", "model"], +} + + def make_default_module_completer() -> ModuleCompleter: # Inside pyrepl, __package__ is set to None by default return ModuleCompleter(namespace={'__package__': None}) @@ -99,8 +108,14 @@ def _find_modules(self, path: str, prefix: str) -> list[str]: modules = [mod_info for mod_info in modules if mod_info.ispkg and mod_info.name == segment] modules = self.iter_submodules(modules) - return [module.name for module in modules - if self.is_suggestion_match(module.name, prefix)] + + module_names = [module.name for module in modules] + try: + module_names += HARDCODED_SUBMODULES[path] + except KeyError: + pass + return [module_name for module_name in module_names + if self.is_suggestion_match(module_name, prefix)] def is_suggestion_match(self, module_name: str, prefix: str) -> bool: if prefix: diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 8e4450fdf99ecd..dec747a8c25d3e 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1,3 +1,4 @@ +import importlib import io import itertools import os @@ -26,7 +27,8 @@ code_to_events, ) from _pyrepl.console import Event -from _pyrepl._module_completer import ImportParser, ModuleCompleter +from _pyrepl._module_completer import (ImportParser, ModuleCompleter, + HARDCODED_SUBMODULES) from _pyrepl.readline import (ReadlineAlikeReader, ReadlineConfig, _ReadlineWrapper) from _pyrepl.readline import multiline_input as readline_multiline_input @@ -930,7 +932,6 @@ def test_func(self): class TestPyReplModuleCompleter(TestCase): def setUp(self): - import importlib # Make iter_modules() search only the standard library. # This makes the test more reliable in case there are # other user packages/scripts on PYTHONPATH which can @@ -1013,14 +1014,6 @@ def test_sub_module_private_completions(self): self.assertEqual(output, expected) def test_builtin_completion_top_level(self): - import importlib - # Make iter_modules() search only the standard library. - # This makes the test more reliable in case there are - # other user packages/scripts on PYTHONPATH which can - # intefere with the completions. - lib_path = os.path.dirname(importlib.__path__[0]) - sys.path = [lib_path] - cases = ( ("import bui\t\n", "import builtins"), ("from bui\t\n", "from builtins"), @@ -1076,6 +1069,20 @@ def test_no_fallback_on_regular_completion(self): output = reader.readline() self.assertEqual(output, expected) + def test_hardcoded_stdlib_submodules(self): + cases = ( + ("import collections.\t\n", "import collections.abc"), + ("from os import \t\n", "from os import path"), + ("import xml.parsers.expat.\t\te\t\n\n", "import xml.parsers.expat.errors"), + ("from xml.parsers.expat import \t\tm\t\n\n", "from xml.parsers.expat import model"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + def test_get_path_and_prefix(self): cases = ( ('', ('', '')), @@ -1204,6 +1211,19 @@ def test_parse_error(self): with self.subTest(code=code): self.assertEqual(actual, None) + +class TestHardcodedSubmodules(TestCase): + def test_hardcoded_stdlib_submodules_are_importable(self): + for parent_path, submodules in HARDCODED_SUBMODULES.items(): + for module_name in submodules: + path = f"{parent_path}.{module_name}" + with self.subTest(path=path): + # We can't use importlib.util.find_spec here, + # since some hardcoded submodules parents are + # not proper packages + importlib.import_module(path) + + class TestPasteEvent(TestCase): def prepare_reader(self, events): console = FakeConsole(events) From 6f38649d043b56e871411f2a6065afcb26e192e1 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 30 Aug 2025 17:15:06 +0000 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2025-08-30-17-15-05.gh-issue-69605.KjBk99.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-08-30-17-15-05.gh-issue-69605.KjBk99.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-30-17-15-05.gh-issue-69605.KjBk99.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-30-17-15-05.gh-issue-69605.KjBk99.rst new file mode 100644 index 00000000000000..7a5108376d2a2b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-30-17-15-05.gh-issue-69605.KjBk99.rst @@ -0,0 +1 @@ +Fix some standard library sumbodules missing from the :term:`REPL` auto-completion of imports. From 362e821a816a7843990f328677f0d08494e72892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Simon?= Date: Sat, 30 Aug 2025 19:26:35 +0200 Subject: [PATCH 3/6] Review feedback --- Lib/_pyrepl/_module_completer.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index 5a2037a66b81fe..9c50ac819ed1f3 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -19,9 +19,9 @@ HARDCODED_SUBMODULES = { # Standard library submodules that are not detected by pkgutil.iter_modules # but can be imported, so should be proposed in completion - "collections": ["abc"], - "os": ["path"], - "xml.parsers.expat": ["errors", "model"], + "collections": ("abc",), + "os": ("path",), + "xml.parsers.expat": ("errors", "model"), } @@ -109,11 +109,8 @@ def _find_modules(self, path: str, prefix: str) -> list[str]: if mod_info.ispkg and mod_info.name == segment] modules = self.iter_submodules(modules) - module_names = [module.name for module in modules] - try: - module_names += HARDCODED_SUBMODULES[path] - except KeyError: - pass + module_names = ([module.name for module in modules] + + HARDCODED_SUBMODULES.get(path, [])) return [module_name for module_name in module_names if self.is_suggestion_match(module_name, prefix)] From a952c2a6c8d589dd1e0af2cc11d3fecd001b807a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Simon?= Date: Sat, 30 Aug 2025 19:38:38 +0200 Subject: [PATCH 4/6] Use .extend instead of + --- Lib/_pyrepl/_module_completer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index 9c50ac819ed1f3..f5781b66026ee6 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -109,8 +109,8 @@ def _find_modules(self, path: str, prefix: str) -> list[str]: if mod_info.ispkg and mod_info.name == segment] modules = self.iter_submodules(modules) - module_names = ([module.name for module in modules] - + HARDCODED_SUBMODULES.get(path, [])) + module_names = [module.name for module in modules] + module_names.extend(HARDCODED_SUBMODULES.get(path, ())) return [module_name for module_name in module_names if self.is_suggestion_match(module_name, prefix)] From fbfe884e66a79de1df33768836a6632b044da991 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Simon?= Date: Sun, 31 Aug 2025 16:12:55 +0200 Subject: [PATCH 5/6] Do not propose hardcoded submodules if local import --- Lib/_pyrepl/_module_completer.py | 17 ++++++++++++++++- Lib/test/test_pyrepl/test_pyrepl.py | 12 ++++++++++++ ...025-08-30-17-15-05.gh-issue-69605.KjBk99.rst | 2 +- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index f5781b66026ee6..ceb1bb3f959e2d 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -1,9 +1,12 @@ from __future__ import annotations +import importlib +import os import pkgutil import sys import token import tokenize +from importlib.machinery import FileFinder from io import StringIO from contextlib import contextmanager from dataclasses import dataclass @@ -50,6 +53,7 @@ def __init__(self, namespace: Mapping[str, Any] | None = None) -> None: self.namespace = namespace or {} self._global_cache: list[pkgutil.ModuleInfo] = [] self._curr_sys_path: list[str] = sys.path[:] + self._stdlib_path = os.path.dirname(importlib.__path__[0]) def get_completions(self, line: str) -> list[str] | None: """Return the next possible import completions for 'line'.""" @@ -104,16 +108,27 @@ def _find_modules(self, path: str, prefix: str) -> list[str]: return [] modules: Iterable[pkgutil.ModuleInfo] = self.global_cache + is_stdlib_import: bool | None = None for segment in path.split('.'): modules = [mod_info for mod_info in modules if mod_info.ispkg and mod_info.name == segment] + if is_stdlib_import is None: + # Top-level import decide if we import from stdlib or not + is_stdlib_import = all( + self._is_stdlib_module(mod_info) for mod_info in modules + ) modules = self.iter_submodules(modules) module_names = [module.name for module in modules] - module_names.extend(HARDCODED_SUBMODULES.get(path, ())) + if is_stdlib_import: + module_names.extend(HARDCODED_SUBMODULES.get(path, ())) return [module_name for module_name in module_names if self.is_suggestion_match(module_name, prefix)] + def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool: + return (isinstance(module_info.module_finder, FileFinder) + and module_info.module_finder.path == self._stdlib_path) + def is_suggestion_match(self, module_name: str, prefix: str) -> bool: if prefix: return module_name.startswith(prefix) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index dec747a8c25d3e..02c7ad0cad68b6 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1083,6 +1083,18 @@ def test_hardcoded_stdlib_submodules(self): output = reader.readline() self.assertEqual(output, expected) + def test_hardcoded_stdlib_submodules_not_proposed_if_local_import(self): + with tempfile.TemporaryDirectory() as _dir: + dir = pathlib.Path(_dir) + (dir / "collections").mkdir() + (dir / "collections" / "__init__.py").touch() + (dir / "collections" / "foo.py").touch() + with patch.object(sys, "path", [dir, *sys.path]): + events = code_to_events("import collections.\t\n") + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, "import collections.foo") + def test_get_path_and_prefix(self): cases = ( ('', ('', '')), diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-30-17-15-05.gh-issue-69605.KjBk99.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-30-17-15-05.gh-issue-69605.KjBk99.rst index 7a5108376d2a2b..d855470fc2b326 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-30-17-15-05.gh-issue-69605.KjBk99.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-30-17-15-05.gh-issue-69605.KjBk99.rst @@ -1 +1 @@ -Fix some standard library sumbodules missing from the :term:`REPL` auto-completion of imports. +Fix some standard library submodules missing from the :term:`REPL` auto-completion of imports. From 985bb20b459dc7ebc69bf421e66cdc787118a74a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 15 Sep 2025 17:26:58 +0200 Subject: [PATCH 6/6] Nit nittety nit --- Lib/_pyrepl/_module_completer.py | 6 +++--- Lib/test/test_pyrepl/test_pyrepl.py | 14 ++++++++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index ceb1bb3f959e2d..cf59e007f4df80 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -22,9 +22,9 @@ HARDCODED_SUBMODULES = { # Standard library submodules that are not detected by pkgutil.iter_modules # but can be imported, so should be proposed in completion - "collections": ("abc",), - "os": ("path",), - "xml.parsers.expat": ("errors", "model"), + "collections": ["abc"], + "os": ["path"], + "xml.parsers.expat": ["errors", "model"], } diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 02c7ad0cad68b6..47d384a209e9ac 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -27,10 +27,16 @@ code_to_events, ) from _pyrepl.console import Event -from _pyrepl._module_completer import (ImportParser, ModuleCompleter, - HARDCODED_SUBMODULES) -from _pyrepl.readline import (ReadlineAlikeReader, ReadlineConfig, - _ReadlineWrapper) +from _pyrepl._module_completer import ( + ImportParser, + ModuleCompleter, + HARDCODED_SUBMODULES, +) +from _pyrepl.readline import ( + ReadlineAlikeReader, + ReadlineConfig, + _ReadlineWrapper, +) from _pyrepl.readline import multiline_input as readline_multiline_input try: