Add support for generating protobuf compliant DESCRIPTORs (#112)

* Add support for generate protobuf compliant DESCRIPTORs

- Fixes danielgtaylor/python-betterproto#443
- Fixes #70

* fix for dependent pkgs

* fix multiple descriptors

* driveby

* comment

* order imports

* fix nested descriptors and stop mutating file input

* lint

* fix

* fix default groups

* Revert "fix default groups"

This reverts commit a85929c.

* better descriptor

* update readme

* pr changes

* lint

* remove import_suffix

* remove source code info from descriptors

Author: edaniels

.github/workflows/ci.yml

@@ -49,7 +49,7 @@ jobs:
 
       - name: Move compiled files to betterproto2
         shell: bash
-        run: mv betterproto2_compiler/tests/output_betterproto betterproto2_compiler/tests/output_betterproto_pydantic betterproto2_compiler/tests/output_reference betterproto2/tests 
+        run: mv betterproto2_compiler/tests/output_betterproto betterproto2_compiler/tests/output_betterproto_pydantic betterproto2_compiler/tests/output_betterproto_descriptor betterproto2_compiler/tests/output_reference betterproto2/tests
 
       - name: Execute test suite
         working-directory: ./betterproto2

.gitignore

@@ -6,7 +6,7 @@
 .pytest_cache
 .python-version
 build/
-tests/output_*
+*/tests/output_*
 **/__pycache__
 dist
 **/*.egg-info

betterproto2/docs/descriptors.md

@@ -0,0 +1,11 @@
+# Google Protobuf Descriptors
+
+Google's protoc plugin for Python generated DESCRIPTOR fields that enable reflection capabilities in many libraries (e.g. grpc, grpclib, mcap).
+
+By default, betterproto2 doesn't generate these as it introduces a dependency on `protobuf`. If you're okay with this dependency and want to generate DESCRIPTORs, use the compiler option `python_betterproto2_opt=google_protobuf_descriptors`.
+
+
+## grpclib Reflection
+
+In order to properly use reflection right now, you will need to modify the `DescriptorPool` that is used by grpclib's `ServerReflection`. To do so, take a look at the use of `ServerReflection.extend` in the `test_grpclib_reflection` test in https://github.com/vmagamedov/grpclib/blob/master/tests/grpc/test_grpclib_reflection.py
+ In the future, once https://github.com/vmagamedov/grpclib/pull/204 is merged, you will be able to pass the `default_google_proto_descriptor_pool` into the `ServerReflection.extend` class method.

betterproto2/mkdocs.yml

@@ -14,6 +14,7 @@ nav:
     - Clients: tutorial/clients.md
   - API: api.md
   - Development: development.md
+  - Protobuf Descriptors: descriptors.md
 
 
 plugins:

betterproto2/pyproject.toml

@@ -22,7 +22,8 @@ Repository = "https://github.com/betterproto/python-betterproto2"
 grpcio = ["grpcio>=1.72.1"]
 grpclib = ["grpclib>=0.4.8"]
 pydantic = ["pydantic>=2.11.5"]
-all = ["grpclib>=0.4.8", "grpcio>=1.72.1", "pydantic>=2.11.5"]
+protobuf = ["protobuf>=5.29.3"]
+all = ["grpclib>=0.4.8", "grpcio>=1.72.1", "pydantic>=2.11.5", "protobuf>=5.29.3"]
 
 [dependency-groups]
 dev = [
@@ -38,7 +39,6 @@ dev = [
 test = [
     "cachelib>=0.13.0",
     "poethepoet>=0.34.0",
-    "protobuf>=5.29.3",
     "pytest>=8.4.0",
     "pytest-asyncio>=1.0.0",
     "pytest-cov>=6.1.1",
@@ -144,6 +144,7 @@ rm -rf tests/output_* &&
 git clone https://github.com/betterproto/python-betterproto2-compiler --branch compiled-test-files --single-branch compiled_files &&
 mv compiled_files/tests_betterproto tests/output_betterproto &&
 mv compiled_files/tests_betterproto_pydantic tests/output_betterproto_pydantic &&
+mv compiled_files/tests_betterproto_pydantic tests/output_betterproto_descriptor &&
 mv compiled_files/tests_reference tests/output_reference &&
 rm -rf compiled_files
 """

betterproto2/src/betterproto2/__init__.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-__all__ = ["__version__", "check_compiler_version", "unwrap", "MessagePool", "validators"]
+__all__ = ["__version__", "check_compiler_version", "classproperty", "unwrap", "MessagePool", "validators"]
 
 import dataclasses
 import enum as builtin_enum

betterproto2/tests/grpc/test_grpclib_reflection.py

@@ -0,0 +1,98 @@
+import asyncio
+from typing import Generic, TypeVar
+
+import pytest
+from google.protobuf import descriptor_pb2
+from grpclib.reflection.service import ServerReflection
+from grpclib.reflection.v1.reflection_grpc import ServerReflectionBase as ServerReflectionBaseV1
+from grpclib.reflection.v1alpha.reflection_grpc import ServerReflectionBase as ServerReflectionBaseV1Alpha
+from grpclib.testing import ChannelFor
+
+from tests.output_betterproto.example_service import TestBase
+from tests.output_betterproto.grpc.reflection.v1 import (
+    ErrorResponse,
+    ListServiceResponse,
+    ServerReflectionRequest,
+    ServerReflectionStub,
+    ServiceResponse,
+)
+from tests.output_betterproto_descriptor.google_proto_descriptor_pool import default_google_proto_descriptor_pool
+
+
+class TestService(TestBase):
+    pass
+
+
+T = TypeVar("T")
+
+
+class AsyncIterableQueue(Generic[T]):
+    CLOSED_SENTINEL = object()
+
+    def __init__(self):
+        self._queue = asyncio.Queue()
+        self._done = asyncio.Event()
+
+    def put(self, item: T):
+        self._queue.put_nowait(item)
+
+    def close(self):
+        self._queue.put_nowait(self.CLOSED_SENTINEL)
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self) -> T:
+        val = await self._queue.get()
+        if val is self.CLOSED_SENTINEL:
+            raise StopAsyncIteration
+        return val
+
+
+@pytest.mark.asyncio
+async def test_grpclib_reflection():
+    service = TestService()
+    services = ServerReflection.extend([service])
+    for service in services:
+        # This won't be needed once https://github.com/vmagamedov/grpclib/pull/204 is in.
+        if isinstance(service, ServerReflectionBaseV1Alpha | ServerReflectionBaseV1):
+            service._pool = default_google_proto_descriptor_pool
+
+    async with ChannelFor(services) as channel:
+        requests = AsyncIterableQueue[ServerReflectionRequest]()
+        responses = ServerReflectionStub(channel).server_reflection_info(requests)
+
+        # list services
+        requests.put(ServerReflectionRequest(list_services=""))
+        response = await anext(responses)
+        assert response.list_services_response == ListServiceResponse(
+            service=[ServiceResponse(name="example_service.Test")]
+        )
+
+        # list methods
+
+        # should fail before we've added descriptors to the protobuf pool
+        requests.put(ServerReflectionRequest(file_containing_symbol="example_service.Test"))
+        response = await anext(responses)
+        assert response.error_response == ErrorResponse(error_code=5, error_message="not found")
+        assert response.file_descriptor_response is None
+
+        # now it should work
+        import tests.output_betterproto_descriptor.example_service as example_service_with_desc
+
+        requests.put(ServerReflectionRequest(file_containing_symbol="example_service.Test"))
+        response = await anext(responses)
+        expected = descriptor_pb2.FileDescriptorProto.FromString(
+            example_service_with_desc.EXAMPLE_SERVICE_PROTO_DESCRIPTOR.serialized_pb
+        )
+        assert response.error_response is None
+        assert response.file_descriptor_response is not None
+        assert len(response.file_descriptor_response.file_descriptor_proto) == 1
+        actual = descriptor_pb2.FileDescriptorProto.FromString(
+            response.file_descriptor_response.file_descriptor_proto[0]
+        )
+        assert actual == expected
+
+        requests.close()
+
+        await anext(responses, None)

betterproto2/tests/grpc/test_message_enum_descriptors.py

@@ -0,0 +1,19 @@
+import pytest
+
+from tests.output_betterproto.import_cousin_package_same_name.test.subpackage import Test
+
+# importing the cousin should cause no descriptor pool errors since the subpackage imports it once already
+from tests.output_betterproto_descriptor.import_cousin_package_same_name.cousin.subpackage import CousinMessage
+from tests.output_betterproto_descriptor.import_cousin_package_same_name.test.subpackage import Test as TestWithDesc
+
+
+def test_message_enum_descriptors():
+    # Normally descriptors are not available as they require protobuf support
+    # to inteoperate with other libraries.
+    with pytest.raises(AttributeError):
+        Test.DESCRIPTOR.full_name
+
+    # But the python_betterproto2_opt=google_protobuf_descriptors option
+    # will add them in as long as protobuf is depended on.
+    assert TestWithDesc.DESCRIPTOR.full_name == "import_cousin_package_same_name.test.subpackage.Test"
+    assert CousinMessage.DESCRIPTOR.full_name == "import_cousin_package_same_name.cousin.subpackage.CousinMessage"

betterproto2/tests/test_deprecated.py

@@ -7,6 +7,7 @@
     Empty,
     Message,
     Test,
+    TestNested,
     TestServiceStub,
 )
 
@@ -26,6 +27,14 @@ def test_deprecated_message():
     assert str(record[0].message) == f"{Message.__name__} is deprecated"
 
 
+def test_deprecated_nested_message_field():
+    with pytest.warns(DeprecationWarning) as record:
+        TestNested(nested_value="hello")
+
+    assert len(record) == 1
+    assert str(record[0].message) == f"TestNested.nested_value is deprecated"
+
+
 def test_message_with_deprecated_field(message):
     with pytest.warns(DeprecationWarning) as record:
         Test(message=message, value=10)