Skip to content

Commit b8af1a3

Browse files
swathipilswathipil
authored
[SchemaRegistry] regenerate + update typing (Azure#31085)
* regen * update swagger readme * regen * refactor sync * add message to potential test proxy error messages * update async based on regen * add more typing * update avro samples * avro turn on mypy/samples * fix mypy * lint/black * lint * lint * adding autorest fix for pre-serialization of error response bug * address kashif's comments * libba/kashif comments * fix protocol import
1 parent 426ef88 commit b8af1a3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+655
-1199
lines changed

‎sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_abstract_avro_encoder.py‎

Lines changed: 32 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,29 @@
33
# Licensed under the MIT License. See License.txt in the project root for license information.
44
# --------------------------------------------------------------------------------------------
55

6-
from typing import BinaryIO, TypeVar, Union, Any
76
from abc import abstractmethod
7+
from typing import BinaryIO, Dict, Union, Any, Mapping
8+
from typing_extensions import Protocol
89

9-
ObjectType = TypeVar("ObjectType")
10+
class AvroDataReader(Protocol):
11+
"""
12+
An Avro Data Reader.
13+
"""
14+
def __init__(self, writers_schema: Any, readers_schema: Any) -> None:
15+
"""
16+
Data reader used as defined in the Avro specification.
17+
:param writers_schema: The writer's schema.
18+
:type writers_schema: any
19+
:param readers_schema: The reader's schema.
20+
:type readers_schema: any
21+
"""
1022

23+
def read(self, decoder: Any, **kwargs: Any) -> object:
24+
"""
25+
Reads data using the provided writer's and reader's schema.
26+
:param decoder: The binary decoder used for reading.
27+
:type decoder: any
28+
"""
1129

1230
class AbstractAvroObjectEncoder(object):
1331
"""
@@ -17,9 +35,8 @@ class AbstractAvroObjectEncoder(object):
1735
@abstractmethod
1836
def get_schema_fullname(
1937
self,
20-
schema,
21-
):
22-
# type: (str) -> str
38+
schema: str,
39+
) -> str:
2340
"""
2441
Returns the namespace-qualified name of the provided schema.
2542
Schema must be a Avro RecordSchema:
@@ -32,31 +49,28 @@ def get_schema_fullname(
3249
@abstractmethod
3350
def encode(
3451
self,
35-
content,
36-
schema,
37-
):
38-
# type: (ObjectType, str) -> bytes
52+
content: Mapping[str, Any],
53+
schema: str,
54+
) -> bytes:
3955
"""Convert the provided value to it's binary representation and write it to the stream.
4056
Schema must be a Avro RecordSchema:
4157
https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema
42-
:param content: An object to encode
43-
:type content: ObjectType
58+
:param content: A mapping to encode
59+
:type content: mapping[str, any]
4460
:param schema: An Avro RecordSchema
4561
:type schema: str
4662
:returns: Encoded bytes
4763
:rtype: bytes
4864
"""
4965

5066
@abstractmethod
51-
def decode(self, content: Union[bytes, BinaryIO], reader: Any):
67+
def decode(self, content: Union[bytes, BinaryIO], reader: AvroDataReader) -> Dict[str, Any]:
5268
"""Read the binary representation into a specific type.
5369
Return type will be ignored, since the schema is deduced from the provided bytes.
5470
:param content: A stream of bytes or bytes directly
5571
:type content: BinaryIO or bytes
56-
:param schema: An Avro RecordSchema
57-
:type schema: str
58-
:keyword readers_schema: An optional reader's schema as defined by the Apache Avro specification.
59-
:paramtype readers_schema: str or None
60-
:returns: An instantiated object
61-
:rtype: ObjectType
72+
:param reader: The data reader.
73+
:type reader: AvroDataReader
74+
:returns: The content dict.
75+
:rtype: dict[str, any]
6276
"""

‎sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_apache_avro_encoder.py‎

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,17 @@
44
# --------------------------------------------------------------------------------------------
55

66
from functools import lru_cache
7-
from typing import BinaryIO, Union, TypeVar, cast
7+
from typing import BinaryIO, Union, cast, IO, Dict, Any, Mapping, TYPE_CHECKING
88
from io import BytesIO
99
import avro # type: ignore
1010
from avro.io import DatumWriter, DatumReader, BinaryDecoder, BinaryEncoder # type: ignore
11-
1211
from ._abstract_avro_encoder import ( # pylint: disable=import-error
1312
AbstractAvroObjectEncoder,
13+
AvroDataReader
1414
)
1515

16-
ObjectType = TypeVar("ObjectType")
16+
if TYPE_CHECKING:
17+
from avro import schema
1718

1819

1920
class ApacheAvroObjectEncoder(AbstractAvroObjectEncoder):
@@ -47,14 +48,14 @@ def get_schema_reader(
4748

4849
def encode(
4950
self,
50-
content, # type: ObjectType
51-
schema, # type: Union[str, bytes, avro.schema.Schema]
51+
content: Mapping[str, Any],
52+
schema: Union[str, bytes, "schema.Schema"],
5253
) -> bytes:
5354
"""Convert the provided value to it's binary representation and write it to the stream.
5455
Schema must be a Avro RecordSchema:
5556
https://avro.apache.org/docs/1.10.0/gettingstartedpython.html#Defining+a+schema
56-
:param content: An object to encode
57-
:type content: ObjectType
57+
:param content: A mapping to encode
58+
:type content: mapping[str, any]
5859
:param schema: An Avro RecordSchema
5960
:type schema: str
6061
:returns: Encoded bytes
@@ -73,26 +74,24 @@ def encode(
7374

7475
def decode(
7576
self,
76-
content, # type: Union[bytes, BinaryIO]
77-
reader, # type: DatumReader
78-
) -> ObjectType:
77+
content: Union[bytes, BinaryIO],
78+
reader: AvroDataReader,
79+
) -> Dict[str, Any]:
7980
"""Read the binary representation into a specific type.
8081
Return type will be ignored, since the schema is deduced from the provided bytes.
8182
:param content: A stream of bytes or bytes directly
8283
:type content: BinaryIO or bytes
83-
:param schema: An Avro RecordSchema
84-
:type schema: str
85-
:keyword readers_schema: An optional reader's schema as defined by the Apache Avro specification.
86-
:paramtype readers_schema: str or None
87-
:returns: An instantiated object
88-
:rtype: ObjectType
84+
:param reader: The Apache Avro data reader.
85+
:type reader: DatumReader
86+
:returns: The content dict.
87+
:rtype: dict[str, any]
8988
"""
9089
if not hasattr(content, "read"):
9190
content = cast(bytes, content)
9291
content = BytesIO(content)
9392

9493
with content: # type: ignore
95-
bin_decoder = BinaryDecoder(content)
96-
decoded_content = reader.read(bin_decoder)
94+
bin_decoder = BinaryDecoder(cast(IO[bytes], content))
95+
decoded_content = cast(Dict[str, Any], reader.read(bin_decoder))
9796

9897
return decoded_content

‎sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_schema_registry_avro_encoder.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import logging
2727
from functools import lru_cache
2828
from typing import (
29+
cast,
2930
TYPE_CHECKING,
3031
Any,
3132
Dict,
@@ -115,7 +116,7 @@ def _get_schema_id(self, schema_name: str, schema_str: str, **kwargs: Any) -> st
115116
:rtype: str
116117
"""
117118
schema_id = self._auto_register_schema_func(
118-
self._schema_group, schema_name, schema_str, "Avro", **kwargs
119+
cast(str, self._schema_group), schema_name, schema_str, "Avro", **kwargs
119120
).id
120121
return schema_id
121122

@@ -186,7 +187,8 @@ def encode(
186187
azure.schemaregistry.encoder.avroencoder.MessageType protocol.
187188
:paramtype message_type: Type[MessageType] or None
188189
:keyword request_options: The keyword arguments for http requests to be passed to the client.
189-
:paramtype request_options: Dict[str, Any]
190+
:paramtype request_options: dict[str, Any] or None
191+
:return: The Message object or the TypedDict containing encoded content and content type.
190192
:rtype: MessageType or MessageContent
191193
:raises ~azure.schemaregistry.encoder.avroencoder.InvalidSchemaError:
192194
Indicates an issue with validating schema.
@@ -232,7 +234,7 @@ def decode(
232234
message: Union[MessageContent, MessageType],
233235
*,
234236
readers_schema: Optional[str] = None,
235-
request_options: Dict[str, Any] = None,
237+
request_options: Optional[Dict[str, Any]] = None,
236238
**kwargs: Any,
237239
) -> Dict[str, Any]:
238240
"""Decode bytes content using schema ID in the content type field. `message` must be one of the following:
@@ -247,8 +249,9 @@ def decode(
247249
:keyword readers_schema: An optional reader's schema as defined by the Apache Avro specification.
248250
:paramtype readers_schema: str or None
249251
:keyword request_options: The keyword arguments for http requests to be passed to the client.
250-
:paramtype request_options: Dict[str, Any]
251-
:rtype: Dict[str, Any]
252+
:paramtype request_options: dict[str, Any] or None
253+
:return: The decoded content.
254+
:rtype: dict[str, Any]
252255
:raises ~azure.schemaregistry.encoder.avroencoder.InvalidSchemaError:
253256
Indicates an issue with validating schemas.
254257
:raises ~azure.schemaregistry.encoder.avroencoder.InvalidContentError:

‎sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/_utils.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ def decode_content(
147147
) from exc
148148

149149
try:
150-
dict_value = avro_encoder.decode(content, reader) # type: Dict[str, Any]
150+
dict_value: Dict[str, Any]= avro_encoder.decode(content, reader)
151151
except SchemaResolutionException as exc:
152152
raise InvalidSchemaError(
153153
f"Incompatible schemas.\nWriter's Schema: {schema_definition}\nReader's Schema: {readers_schema}",

‎sdk/schemaregistry/azure-schemaregistry-avroencoder/azure/schemaregistry/encoder/avroencoder/aio/_schema_registry_avro_encoder_async.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
#
2525
# --------------------------------------------------------------------------
2626
import logging
27-
from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, overload, Type, Union
27+
from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, overload, Type, Union, cast
2828
from .._utils import ( # pylint: disable=import-error
2929
validate_schema,
3030
create_message_content,
@@ -105,7 +105,7 @@ async def _get_schema_id(self, schema_name: str, schema_str: str, **kwargs: Any)
105105
:rtype: str
106106
"""
107107
schema_properties = await self._auto_register_schema_func(
108-
self._schema_group, schema_name, schema_str, "Avro", **kwargs
108+
cast(str, self._schema_group), schema_name, schema_str, "Avro", **kwargs
109109
)
110110
return schema_properties.id
111111

@@ -176,6 +176,7 @@ async def encode(
176176
:paramtype message_type: Type[MessageType] or None
177177
:keyword request_options: The keyword arguments for http requests to be passed to the client.
178178
:paramtype request_options: Dict[str, Any]
179+
:return: The Message object or the TypedDict containing encoded content and content type.
179180
:rtype: MessageType or MessageContent
180181
:raises ~azure.schemaregistry.encoder.avroencoder.InvalidSchemaError:
181182
Indicates an issue with validating schema.
@@ -220,7 +221,7 @@ async def decode(
220221
message: Union[MessageContent, MessageType],
221222
*,
222223
readers_schema: Optional[str] = None,
223-
request_options: Dict[str, Any] = None,
224+
request_options: Optional[Dict[str, Any]] = None,
224225
**kwargs: Any,
225226
) -> Dict[str, Any]:
226227
"""Decode bytes content using schema ID in the content type field. `message` must be one of the following:
@@ -235,8 +236,9 @@ async def decode(
235236
:keyword readers_schema: An optional reader's schema as defined by the Apache Avro specification.
236237
:paramtype readers_schema: str or None
237238
:keyword request_options: The keyword arguments for http requests to be passed to the client.
238-
:paramtype request_options: Dict[str, Any]
239-
:rtype: Dict[str, Any]
239+
:paramtype request_options: dict[str, Any] or None
240+
:return: The decoded content.
241+
:rtype: dict[str, Any]
240242
:raises ~azure.schemaregistry.encoder.avroencoder.InvalidSchemaError:
241243
Indicates an issue with validating schemas.
242244
:raises ~azure.schemaregistry.encoder.avroencoder.InvalidContentError:

‎sdk/schemaregistry/azure-schemaregistry-avroencoder/pyproject.toml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
[tool.azure-sdk-build]
2-
mypy = false
2+
mypy = true
33
pyright = false
4-
type_check_samples = false
4+
type_check_samples = true
55
verifytypes = false

‎sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/encode_and_decode_event_data_message_async.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
1) AZURE_TENANT_ID - The ID of the service principal's tenant. Also called its 'directory' ID.
3939
2) AZURE_CLIENT_ID - The service principal's client ID. Also called its 'application' ID.
4040
3) AZURE_CLIENT_SECRET - One of the service principal's client secrets.
41-
4) SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE - The schema registry fully qualified namespace,
41+
4) SCHEMAREGISTRY_AVRO_FULLY_QUALIFIED_NAMESPACE - The schema registry fully qualified namespace,
4242
which should follow the format: `<your-namespace>.servicebus.windows.net`
4343
5) SCHEMAREGISTRY_GROUP - The name of the schema group.
4444
@@ -58,7 +58,7 @@
5858
CLIENT_ID=os.environ['AZURE_CLIENT_ID']
5959
CLIENT_SECRET=os.environ['AZURE_CLIENT_SECRET']
6060

61-
SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE=os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE']
61+
SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE=os.environ['SCHEMAREGISTRY_AVRO_FULLY_QUALIFIED_NAMESPACE']
6262
GROUP_NAME=os.environ['SCHEMAREGISTRY_GROUP']
6363
SCHEMA_STRING = """
6464
{"namespace": "example.avro",

‎sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/encode_and_decode_with_message_content_async.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
1) AZURE_TENANT_ID - The ID of the service principal's tenant. Also called its 'directory' ID.
3939
2) AZURE_CLIENT_ID - The service principal's client ID. Also called its 'application' ID.
4040
3) AZURE_CLIENT_SECRET - One of the service principal's client secrets.
41-
4) SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE - The schema registry fully qualified namespace,
41+
4) SCHEMAREGISTRY_AVRO_FULLY_QUALIFIED_NAMESPACE - The schema registry fully qualified namespace,
4242
which should follow the format: `<your-namespace>.servicebus.windows.net`
4343
5) SCHEMAREGISTRY_GROUP - The name of the schema group.
4444
@@ -60,7 +60,7 @@
6060
CLIENT_SECRET = os.environ["AZURE_CLIENT_SECRET"]
6161

6262
SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ[
63-
"SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE"
63+
"SCHEMAREGISTRY_AVRO_FULLY_QUALIFIED_NAMESPACE"
6464
]
6565
GROUP_NAME = os.environ["SCHEMAREGISTRY_GROUP"]
6666
SCHEMA_STRING = """

‎sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/eventhub_receive_integration_async.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
2) AZURE_CLIENT_ID - Required for use of the credential. The service principal's client ID.
1818
Also called its 'application' ID.
1919
3) AZURE_CLIENT_SECRET - Required for use of the credential. One of the service principal's client secrets.
20-
4) SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE - The schema registry fully qualified namespace,
20+
4) SCHEMAREGISTRY_AVRO_FULLY_QUALIFIED_NAMESPACE - The schema registry fully qualified namespace,
2121
which should follow the format: `<your-namespace>.servicebus.windows.net`
2222
5) SCHEMAREGISTRY_GROUP - The name of the schema group.
2323
6) EVENT_HUB_CONN_STR - The connection string of the Event Hubs namespace to receive events from.
@@ -37,7 +37,7 @@
3737
EVENTHUB_CONNECTION_STR = os.environ['EVENT_HUB_CONN_STR']
3838
EVENTHUB_NAME = os.environ['EVENT_HUB_NAME']
3939

40-
SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE']
40+
SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ['SCHEMAREGISTRY_AVRO_FULLY_QUALIFIED_NAMESPACE']
4141
GROUP_NAME = os.environ['SCHEMAREGISTRY_GROUP']
4242

4343

@@ -62,7 +62,7 @@ async def on_event(partition_context, event):
6262
print(f"Received event from partition: {partition_context.partition_id}.")
6363

6464
bytes_payload = b"".join(b for b in event.body)
65-
print(f'The received bytes of the EventData is {bytes_payload}.')
65+
print(f'The received bytes of the EventData is {bytes_payload!r}.')
6666

6767
# Use the decode method to decode the payload of the event.
6868
# The decode method will extract the schema id from the content_type, and automatically retrieve the Avro Schema

‎sdk/schemaregistry/azure-schemaregistry-avroencoder/samples/async_samples/eventhub_send_integration_async.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
2) AZURE_CLIENT_ID - Required for use of the credential. The service principal's client ID.
1818
Also called its 'application' ID.
1919
3) AZURE_CLIENT_SECRET - Required for use of the credential. One of the service principal's client secrets.
20-
4) SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE - The schema registry fully qualified namespace,
20+
4) SCHEMAREGISTRY_AVRO_FULLY_QUALIFIED_NAMESPACE - The schema registry fully qualified namespace,
2121
which should follow the format: `<your-namespace>.servicebus.windows.net`
2222
5) SCHEMAREGISTRY_GROUP - The name of the schema group.
2323
6) EVENT_HUB_CONN_STR - The connection string of the Event Hubs namespace to send events to.
@@ -38,7 +38,7 @@
3838
EVENTHUB_CONNECTION_STR = os.environ['EVENT_HUB_CONN_STR']
3939
EVENTHUB_NAME = os.environ['EVENT_HUB_NAME']
4040

41-
SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ['SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE']
41+
SCHEMAREGISTRY_FULLY_QUALIFIED_NAMESPACE = os.environ['SCHEMAREGISTRY_AVRO_FULLY_QUALIFIED_NAMESPACE']
4242
GROUP_NAME = os.environ['SCHEMAREGISTRY_GROUP']
4343

4444
SCHEMA_STRING = """

0 commit comments

Comments
 (0)