Aidan/pep 702 support (#683)

* Add initial PEP702 support

Signed-off-by: Aidan Jensen <aidan.jensen@robust.ai>

* Pull comments from deprecation option for warning details

Signed-off-by: Aidan Jensen <aidan.jensen@robust.ai>

* Use ast.parse to make sure the comment string is a valid python string

Signed-off-by: Aidan Jensen <aidan.jensen@robust.ai>

---------

Signed-off-by: Aidan Jensen <aidan.jensen@robust.ai>
Co-authored-by: Aidan Jensen <aidan.jensen@robust.ai>

Author: aidandj

.flake8

@@ -2,7 +2,7 @@
 per-file-ignores =
   *.py: E203, E301, E302, E305, E501
   *.pyi: E301, E302, E305, E501, E701, E741, F401, F403, F405, F822, Y037
-  *_pb2.pyi: E301, E302, E305, E501, E701, E741, F401, F403, F405, F822, Y037, Y021
-  *_pb2_grpc.pyi: E301, E302, E305, E501, E701, E741, F401, F403, F405, F822, Y037, Y021, Y023
+  *_pb2.pyi: E301, E302, E305, E501, E701, E741, F401, F403, F405, F822, Y037, Y021, Y053
+  *_pb2_grpc.pyi: E301, E302, E305, E501, E701, E741, F401, F403, F405, F822, Y037, Y021, Y023, Y053
 
 extend_exclude = venv*,*_pb2.py,*_pb2_grpc.py,build/

CHANGELOG.md

@@ -4,6 +4,13 @@
 - Change the top-level mangling prefix from `global___` to `Global___` to respect
   [Y042](https://github.com/PyCQA/flake8-pyi/blob/main/ERRORCODES.md#list-of-warnings) naming convention.
 - Support client stub async typing overloads
+- Support [PEP702](https://peps.python.org/pep-0702/) deprecations
+  - Message deprecations are supported
+  - Field deprecations are not. This may be possible with init overloads
+  - Service deprecations are supported for Sync stubs
+    - Not for async stubs
+  - Enum message deprecation is supported
+    - Enum field deprecation is not
 
 ## 3.6.0
 

README.md

@@ -219,6 +219,16 @@ However, it may be helpful when migrating existing proto2 code, where the distin
 protoc --python_out=output/location --mypy_out=relax_strict_optional_primitives:output/location
 ```
 
+### `use_default_deprecation_warnings`
+
+By default mypy-protobuf will pull the leading and trailing comments from the deprecation option definition,
+and insert it into the deprecation warning. This option will instead use a standard deprecation warning instead of comments.
+
+```
+protoc --python_out=output/location --mypy_out=use_default_deprecation_warning:output/location
+
+```
+
 ### Output suppression
 
 To suppress output, you can run

mypy_protobuf/main.py

@@ -2,6 +2,7 @@
 """Protoc Plugin to generate mypy stubs."""
 from __future__ import annotations
 
+import ast
 import sys
 from collections import defaultdict
 from contextlib import contextmanager
@@ -149,12 +150,14 @@ def __init__(
         descriptors: Descriptors,
         readable_stubs: bool,
         relax_strict_optional_primitives: bool,
+        use_default_deprecation_warnings: bool,
         grpc: bool,
     ) -> None:
         self.fd = fd
         self.descriptors = descriptors
         self.readable_stubs = readable_stubs
         self.relax_strict_optional_primitives = relax_strict_optional_primitives
+        self.use_default_depreaction_warnings = use_default_deprecation_warnings
         self.grpc = grpc
         self.lines: List[str] = []
         self.indent = ""
@@ -165,6 +168,7 @@ def __init__(
         # if {z} is None, then it shortens to `from {x} import {y}`
         self.from_imports: Dict[str, Set[Tuple[str, str | None]]] = defaultdict(set)
         self.typing_extensions_min: Optional[Tuple[int, int]] = None
+        self.deprecated_min: Optional[Tuple[int, int]] = None
 
         # Comments
         self.source_code_info_by_scl = {tuple(location.path): location for location in fd.source_code_info.location}
@@ -180,6 +184,11 @@ def _import(self, path: str, name: str) -> str:
                 self.typing_extensions_min = stabilization[name]
             return "typing_extensions." + name
 
+        if path == "warnings" and name == "deprecated":
+            if not self.deprecated_min or self.deprecated_min < (3, 11):
+                self.deprecated_min = (3, 13)
+            return name
+
         imp = path.replace("/", ".")
         if self.readable_stubs:
             self.from_imports[imp].add((name, None))
@@ -251,6 +260,51 @@ def _has_comments(self, scl: SourceCodeLocation) -> bool:
         sci_loc = self.source_code_info_by_scl.get(tuple(scl))
         return sci_loc is not None and bool(sci_loc.leading_detached_comments or sci_loc.leading_comments or sci_loc.trailing_comments)
 
+    def _get_comments(self, scl: SourceCodeLocation) -> List[str]:
+        """Return list of comment lines"""
+        if not self._has_comments(scl):
+            return []
+
+        sci_loc = self.source_code_info_by_scl.get(tuple(scl))
+        assert sci_loc is not None
+
+        leading_detached_lines = []
+        leading_lines = []
+        trailing_lines = []
+        for leading_detached_comment in sci_loc.leading_detached_comments:
+            leading_detached_lines = self._break_text(leading_detached_comment)
+        if sci_loc.leading_comments is not None:
+            leading_lines = self._break_text(sci_loc.leading_comments)
+        # Trailing comments also go in the header - to make sure it gets into the docstring
+        if sci_loc.trailing_comments is not None:
+            trailing_lines = self._break_text(sci_loc.trailing_comments)
+
+        lines = leading_detached_lines
+        if leading_detached_lines and (leading_lines or trailing_lines):
+            lines.append("")
+        lines.extend(leading_lines)
+        lines.extend(trailing_lines)
+
+        return lines
+
+    def _write_deprecation_warning(self, scl: SourceCodeLocation, default_message: str) -> None:
+        msg = default_message
+        if not self.use_default_depreaction_warnings and (comments := self._get_comments(scl)):
+            # Make sure the comment string is a valid python string literal
+            joined = "\\n".join(comments)
+            # Check that it is valid python string by using ast.parse
+            try:
+                ast.parse(f'"""{joined}"""')
+                msg = joined
+            except SyntaxError as e:
+                print(f"Warning: Deprecation comment {joined} could not be parsed as a python string literal. Using default deprecation message. {e}", file=sys.stderr)
+                pass
+        self._write_line(
+            '@{}("""{}""")',
+            self._import("warnings", "deprecated"),
+            msg,
+        )
+
     def _write_comments(self, scl: SourceCodeLocation) -> bool:
         """Return true if any comments were written"""
         if not self._has_comments(scl):
@@ -364,6 +418,11 @@ def write_enums(
                 )
             wl("")
 
+            if enum.options.deprecated:
+                self._write_deprecation_warning(
+                    scl + [d.EnumDescriptorProto.OPTIONS_FIELD_NUMBER] + [d.EnumOptions.DEPRECATED_FIELD_NUMBER],
+                    "This enum has been marked as deprecated using proto enum options.",
+                )
             if self._has_comments(scl):
                 wl(f"class {class_name}({enum_helper_class}, metaclass={etw_helper_class}):")
                 with self._indent():
@@ -409,6 +468,11 @@ def write_messages(
 
             class_name = desc.name if desc.name not in PYTHON_RESERVED else "_r_" + desc.name
             message_class = self._import("google.protobuf.message", "Message")
+            if desc.options.deprecated:
+                self._write_deprecation_warning(
+                    scl_prefix + [i] + [d.DescriptorProto.OPTIONS_FIELD_NUMBER] + [d.MessageOptions.DEPRECATED_FIELD_NUMBER],
+                    "This message has been marked as deprecated using proto message options.",
+                )
             wl("@{}", self._import("typing", "final"))
             wl(f"class {class_name}({message_class}{addl_base}):")
             with self._indent():
@@ -835,6 +899,11 @@ def write_grpc_services(
             self.write_grpc_type_vars(service)
 
             # The stub client
+            if service.options.deprecated:
+                self._write_deprecation_warning(
+                    scl + [d.ServiceDescriptorProto.OPTIONS_FIELD_NUMBER] + [d.ServiceOptions.DEPRECATED_FIELD_NUMBER],
+                    "This stub has been marked as deprecated using proto service options.",
+                )
             class_name = f"{service.name}Stub"
             wl(
                 "class {}({}[{}]):",
@@ -875,6 +944,11 @@ def write_grpc_services(
             wl("")
 
             # The service definition interface
+            if service.options.deprecated:
+                self._write_deprecation_warning(
+                    scl + [d.ServiceDescriptorProto.OPTIONS_FIELD_NUMBER] + [d.ServiceOptions.DEPRECATED_FIELD_NUMBER],
+                    "This servicer has been marked as deprecated using proto service options.",
+                )
             wl(
                 "class {}Servicer(metaclass={}):",
                 service.name,
@@ -886,6 +960,11 @@ def write_grpc_services(
                 self.write_grpc_methods(service, scl)
             server = self._import("grpc", "Server")
             aserver = self._import("grpc.aio", "Server")
+            if service.options.deprecated:
+                self._write_deprecation_warning(
+                    scl + [d.ServiceDescriptorProto.OPTIONS_FIELD_NUMBER] + [d.ServiceOptions.DEPRECATED_FIELD_NUMBER],
+                    "This servicer has been marked as deprecated using proto service options.",
+                )
             wl(
                 "def add_{}Servicer_to_server(servicer: {}Servicer, server: {}) -> None: ...",
                 service.name,
@@ -1001,7 +1080,7 @@ def write(self) -> str:
                 # n,n to force a reexport (from x import y as y)
                 self.from_imports[reexport_imp].update((n, n) for n in names)
 
-        if self.typing_extensions_min:
+        if self.typing_extensions_min or self.deprecated_min:
             self.imports.add("sys")
         for pkg in sorted(self.imports):
             self._write_line(f"import {pkg}")
@@ -1011,6 +1090,12 @@ def write(self) -> str:
             self._write_line("    import typing as typing_extensions")
             self._write_line("else:")
             self._write_line("    import typing_extensions")
+        if self.deprecated_min:
+            self._write_line("")
+            self._write_line(f"if sys.version_info >= {self.deprecated_min}:")
+            self._write_line("    from warnings import deprecated")
+            self._write_line("else:")
+            self._write_line("    from typing_extensions import deprecated")
 
         for pkg, items in sorted(self.from_imports.items()):
             self._write_line(f"from {pkg} import (")
@@ -1041,13 +1126,15 @@ def generate_mypy_stubs(
     quiet: bool,
     readable_stubs: bool,
     relax_strict_optional_primitives: bool,
+    use_default_deprecation_warnings: bool,
 ) -> None:
     for name, fd in descriptors.to_generate.items():
         pkg_writer = PkgWriter(
             fd,
             descriptors,
             readable_stubs,
             relax_strict_optional_primitives,
+            use_default_deprecation_warnings,
             grpc=False,
         )
 
@@ -1073,13 +1160,15 @@ def generate_mypy_grpc_stubs(
     quiet: bool,
     readable_stubs: bool,
     relax_strict_optional_primitives: bool,
+    use_default_deprecation_warnings: bool,
 ) -> None:
     for name, fd in descriptors.to_generate.items():
         pkg_writer = PkgWriter(
             fd,
             descriptors,
             readable_stubs,
             relax_strict_optional_primitives,
+            use_default_deprecation_warnings,
             grpc=True,
         )
         pkg_writer.write_grpc_async_hacks()
@@ -1131,6 +1220,7 @@ def main() -> None:
             "quiet" in request.parameter,
             "readable_stubs" in request.parameter,
             "relax_strict_optional_primitives" in request.parameter,
+            "use_default_deprecation_warnings" in request.parameter,
         )
 
 
@@ -1143,6 +1233,7 @@ def grpc() -> None:
             "quiet" in request.parameter,
             "readable_stubs" in request.parameter,
             "relax_strict_optional_primitives" in request.parameter,
+            "use_default_deprecation_warnings" in request.parameter,
         )
 
 

mypy_requirements.txt

@@ -1,2 +1,2 @@
 # Requirements to run mypy itself. Mypy executable exists in a separate venv.
-mypy==1.13.0
+mypy==1.14.1

proto/testproto/grpc/dummy.proto

@@ -22,3 +22,22 @@ service DummyService {
   // StreamStream
   rpc StreamStream (stream DummyRequest) returns (stream DummyReply) {}
 }
+
+message DeprecatedRequest {
+  option deprecated = true;
+  string old_field = 1 [deprecated = true];
+}
+
+// Marking the service as deprecated
+service DeprecatedService {
+  // This service is deprecated
+  option deprecated = true;
+  // DeprecatedMethod
+  rpc DeprecatedMethod (DeprecatedRequest) returns (DummyReply) {
+    option deprecated = true;
+  }
+  // DeprecatedMethodNotDeprecatedRequest
+  rpc DeprecatedMethodNotDeprecatedRequest (DummyRequest) returns (DummyReply) {
+    option deprecated = true;
+  }
+}

proto/testproto/test.proto

@@ -180,3 +180,30 @@ message SelfField {
     // Field self -> must generate an __init__ method w/ different name
     optional int64 self = 1;
 }
+
+message DeprecatedMessage {
+    // This message is deprecated
+    option deprecated = true;
+
+    optional string a_string = 1;
+}
+
+message DeprecatedMessageBadComment {
+    // This message is deprecated
+    // """ triple quotes in comment
+    option deprecated = true;
+
+    optional string a_string = 1;
+}
+
+enum DeprecatedEnum {
+    // This enum is deprecated
+    // 2 lines of comments
+    // "Quotes in comments"
+    // and 'single quotes'
+    option deprecated = true;
+    // Trailing comment
+
+    DEPRECATED_ONE = 1;
+    DEPRECATED_TWO = 2;
+}

pyproject.toml

@@ -18,6 +18,8 @@ extend_skip_glob = ["*_pb2.py"]
 [tool.mypy]
 strict = true
 show_error_codes = true
+enable_error_code = ["deprecated"]
+report_deprecated_as_note = true
 
 [tool.pyright]
 venvPath = "."

run_test.sh

@@ -6,10 +6,16 @@ NC='\033[0m'
 PY_VER_MYPY_PROTOBUF=${PY_VER_MYPY_PROTOBUF:=3.11.4}
 PY_VER_MYPY_PROTOBUF_SHORT=$(echo "$PY_VER_MYPY_PROTOBUF" | cut -d. -f1-2)
 PY_VER_MYPY=${PY_VER_MYPY:=3.8.17}
-PY_VER_UNIT_TESTS="${PY_VER_UNIT_TESTS:=3.8.17}"
+PY_VER_UNIT_TESTS="${PY_VER_UNIT_TESTS:=3.8.17 3.13.9 3.14.0}"
+
+
 
 if [ -e "$CUSTOM_TYPESHED_DIR" ]; then
     export MYPYPATH=$CUSTOM_TYPESHED_DIR/stubs/protobuf
+    export CUSTOM_TYPESHED_DIR_ARG="--custom-typeshed-dir=$CUSTOM_TYPESHED_DIR"
+else
+    # mypy does not emit deprecation warnings for typeshed stubs. Setting an empty custom-typeshed-dir was causing the current directory to be considered a typeshed dir, and hiding deprecation warnings.
+    export CUSTOM_TYPESHED_DIR_ARG=""
 fi
 
 # Install protoc
@@ -101,7 +107,7 @@ MYPY_PROTOBUF_VENV=venv_$PY_VER_MYPY_PROTOBUF
 
     # Run mypy on mypy-protobuf internal code for developers to catch issues
     FILES="mypy_protobuf/main.py"
-    "$MYPY_VENV/bin/mypy" --custom-typeshed-dir="$CUSTOM_TYPESHED_DIR" --python-executable="$MYPY_PROTOBUF_VENV/bin/python3" --python-version="$PY_VER_MYPY_PROTOBUF_SHORT" $FILES
+    "$MYPY_VENV/bin/mypy" ${CUSTOM_TYPESHED_DIR_ARG:+"$CUSTOM_TYPESHED_DIR_ARG"} --python-executable="$MYPY_PROTOBUF_VENV/bin/python3" --python-version="$PY_VER_MYPY_PROTOBUF_SHORT" $FILES
 
     # Generate protos
     python --version
@@ -124,6 +130,7 @@ MYPY_PROTOBUF_VENV=venv_$PY_VER_MYPY_PROTOBUF
     find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=quiet:test/generated
     find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=readable_stubs:test/generated
     find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=relax_strict_optional_primitives:test/generated
+    find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=use_default_deprecation_warnings:test/generated
     # Overwrite w/ run with mypy-protobuf without flags
     find proto -name "*.proto" -print0 | xargs -0 "$PROTOC" "${PROTOC_ARGS[@]}" --mypy_out=test/generated
 
@@ -153,13 +160,13 @@ for PY_VER in $PY_VER_UNIT_TESTS; do
 
         # Run mypy
         MODULES=( -m test.test_generated_mypy -m test.test_grpc_usage -m test.test_grpc_async_usage )
-        mypy --custom-typeshed-dir="$CUSTOM_TYPESHED_DIR" --python-executable="$UNIT_TESTS_VENV"/bin/python --python-version="$PY_VER_MYPY_TARGET" "${MODULES[@]}"
+        mypy ${CUSTOM_TYPESHED_DIR_ARG:+"$CUSTOM_TYPESHED_DIR_ARG"} --python-executable="$UNIT_TESTS_VENV"/bin/python --python-version="$PY_VER_MYPY_TARGET" "${MODULES[@]}"
 
         # Run stubtest. Stubtest does not work with python impl - only cpp impl
         pip install -r test_requirements.txt
         API_IMPL="$(python3 -c "import google.protobuf.internal.api_implementation as a ; print(a.Type())")"
         if [[ $API_IMPL != "python" ]]; then
-            PYTHONPATH=test/generated python3 -m mypy.stubtest --custom-typeshed-dir="$CUSTOM_TYPESHED_DIR" --allowlist stubtest_allowlist.txt testproto
+            PYTHONPATH=test/generated python3 -m mypy.stubtest ${CUSTOM_TYPESHED_DIR_ARG:+"$CUSTOM_TYPESHED_DIR_ARG"} --allowlist stubtest_allowlist.txt testproto
         fi
 
         # run mypy on negative-tests (expected mypy failures)
@@ -171,7 +178,7 @@ for PY_VER in $PY_VER_UNIT_TESTS; do
             PY_VER_MYPY_TARGET=$(echo "$1" | cut -d. -f1-2)
             export MYPYPATH=$MYPYPATH:test/generated
             # Use --no-incremental to avoid caching issues: https://github.com/python/mypy/issues/16363
-            mypy --custom-typeshed-dir="$CUSTOM_TYPESHED_DIR" --python-executable="venv_$1/bin/python" --no-incremental --python-version="$PY_VER_MYPY_TARGET" "${@: 2}" > "$MYPY_OUTPUT/mypy_output" || true
+            mypy ${CUSTOM_TYPESHED_DIR_ARG:+"$CUSTOM_TYPESHED_DIR_ARG"} --python-executable="venv_$1/bin/python" --no-incremental --python-version="$PY_VER_MYPY_TARGET" "${@: 2}" > "$MYPY_OUTPUT/mypy_output" || true
             cut -d: -f1,3- "$MYPY_OUTPUT/mypy_output" > "$MYPY_OUTPUT/mypy_output.omit_linenos"
         }