Skip to content

Commit 902abfb

Browse files
m-reddingjsquirechristothes
authored
[Schema Registry] Add pluggable serializer (Azure#36926)
* add code and tests * updates * new tests, doc updates, cleanup * sample update * move Lru cache to core * API gen * md updates * doc * trying to fix merge conflict * feedback * fix * updating snippets * Update sdk/core/Azure.Core/src/Serialization/SchemaValidator.cs Co-authored-by: Jesse Squire <jesse.squire@gmail.com> * Apply suggestions from code review Co-authored-by: Jesse Squire <jesse.squire@gmail.com> * feedback/ regenerating snippets * removing unneeded md links * Update sdk/schemaregistry/Azure.Data.SchemaRegistry/samples/Sample02_Serialization.md Co-authored-by: Christopher Scott <chriscott@hotmail.com> * WIP - feedback part 1 * updates * additional tests, more feedback, api changes support * trying to fix indentation * trying to fix indentations 2 * pipeline fix * test * updating nullable annotations * WIP feedback * feedback 1 * feedback 2 * more feedback * tweaking docs * pipeline fixes * change namespace from Azure.Core.Experimental to just Azure.Core * fixes * update exception logic * update schema validator aggregate exception message * test fix * tweak sample --------- Co-authored-by: Jesse Squire <jesse.squire@gmail.com> Co-authored-by: Christopher Scott <chriscott@hotmail.com>
1 parent 41248f6 commit 902abfb

27 files changed

+2107
-97
lines changed

‎sdk/core/Azure.Core.Experimental/api/Azure.Core.Experimental.net461.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,3 +106,23 @@ public readonly partial struct Value
106106
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]public bool TryGetValue<T>(out T value) { throw null; }
107107
}
108108
}
109+
namespace Azure.Core
110+
{
111+
public partial class LruCache<TKey, TValue> : System.Collections.Generic.IEnumerable<System.Collections.Generic.KeyValuePair<TKey, TValue>>, System.Collections.IEnumerable where TKey : notnull
112+
{
113+
public LruCache(int capacity) { }
114+
public int Count { get { throw null; } }
115+
public int TotalLength { get { throw null; } }
116+
public void AddOrUpdate(TKey key, TValue? val, int length) { }
117+
public System.Collections.Generic.IEnumerator<System.Collections.Generic.KeyValuePair<TKey, TValue>> GetEnumerator() { throw null; }
118+
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { throw null; }
119+
public bool TryGet(TKey key, out TValue? value) { throw null; }
120+
}
121+
public abstract partial class SchemaValidator
122+
{
123+
protected SchemaValidator() { }
124+
public abstract string GenerateSchema(System.Type dataType);
125+
public abstract bool TryValidate(object data, System.Type dataType, string schemaDefinition, out System.Collections.Generic.IEnumerable<System.Exception> validationErrors);
126+
public virtual void Validate(object data, System.Type dataType, string schemaDefinition) { }
127+
}
128+
}

‎sdk/core/Azure.Core.Experimental/api/Azure.Core.Experimental.net6.0.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,3 +106,23 @@ public readonly partial struct Value
106106
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]public bool TryGetValue<T>(out T value) { throw null; }
107107
}
108108
}
109+
namespace Azure.Core
110+
{
111+
public partial class LruCache<TKey, TValue> : System.Collections.Generic.IEnumerable<System.Collections.Generic.KeyValuePair<TKey, TValue>>, System.Collections.IEnumerable where TKey : notnull
112+
{
113+
public LruCache(int capacity) { }
114+
public int Count { get { throw null; } }
115+
public int TotalLength { get { throw null; } }
116+
public void AddOrUpdate(TKey key, TValue? val, int length) { }
117+
public System.Collections.Generic.IEnumerator<System.Collections.Generic.KeyValuePair<TKey, TValue>> GetEnumerator() { throw null; }
118+
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { throw null; }
119+
public bool TryGet(TKey key, out TValue? value) { throw null; }
120+
}
121+
public abstract partial class SchemaValidator
122+
{
123+
protected SchemaValidator() { }
124+
public abstract string GenerateSchema(System.Type dataType);
125+
public abstract bool TryValidate(object data, System.Type dataType, string schemaDefinition, out System.Collections.Generic.IEnumerable<System.Exception> validationErrors);
126+
public virtual void Validate(object data, System.Type dataType, string schemaDefinition) { }
127+
}
128+
}

‎sdk/core/Azure.Core.Experimental/api/Azure.Core.Experimental.netstandard2.0.cs‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,3 +106,23 @@ public readonly partial struct Value
106106
[System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]public bool TryGetValue<T>(out T value) { throw null; }
107107
}
108108
}
109+
namespace Azure.Core
110+
{
111+
public partial class LruCache<TKey, TValue> : System.Collections.Generic.IEnumerable<System.Collections.Generic.KeyValuePair<TKey, TValue>>, System.Collections.IEnumerable where TKey : notnull
112+
{
113+
public LruCache(int capacity) { }
114+
public int Count { get { throw null; } }
115+
public int TotalLength { get { throw null; } }
116+
public void AddOrUpdate(TKey key, TValue? val, int length) { }
117+
public System.Collections.Generic.IEnumerator<System.Collections.Generic.KeyValuePair<TKey, TValue>> GetEnumerator() { throw null; }
118+
System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { throw null; }
119+
public bool TryGet(TKey key, out TValue? value) { throw null; }
120+
}
121+
public abstract partial class SchemaValidator
122+
{
123+
protected SchemaValidator() { }
124+
public abstract string GenerateSchema(System.Type dataType);
125+
public abstract bool TryValidate(object data, System.Type dataType, string schemaDefinition, out System.Collections.Generic.IEnumerable<System.Exception> validationErrors);
126+
public virtual void Validate(object data, System.Type dataType, string schemaDefinition) { }
127+
}
128+
}

‎sdk/core/Azure.Core.Experimental/src/SchemaValidator/LruCache.cs‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System.Collections;
5+
using System.Collections.Generic;
6+
7+
#nullable enable
8+
9+
namespace Azure.Core
10+
{
11+
/// <summary>
12+
/// A simple LRU cache implementation using a doubly linked list and dictionary.
13+
/// </summary>
14+
/// <typeparam name="TKey">The type of key.</typeparam>
15+
/// <typeparam name="TValue">The type of value.</typeparam>
16+
public class LruCache<TKey, TValue> : IEnumerable<KeyValuePair<TKey, TValue>>
17+
where TKey : notnull
18+
{
19+
private readonly int _capacity;
20+
private readonly LinkedList<KeyValuePair<TKey, TValue?>> _linkedList;
21+
private readonly Dictionary<TKey, (LinkedListNode<KeyValuePair<TKey, TValue?>> Node, int Length)> _map;
22+
private readonly object _syncLock;
23+
24+
/// <summary>
25+
/// Gets the number of key/value pairs contained in the <see cref="LruCache{TKey, TValue}"/>.
26+
/// </summary>
27+
public int Count => _linkedList.Count;
28+
29+
/// <summary>
30+
/// Gets the total length of all values currently stored in the <see cref="LruCache{TKey, TValue}"/>.
31+
/// </summary>
32+
public int TotalLength { get; private set; }
33+
34+
/// <summary>
35+
/// Initializes a new instance of the <see cref="LruCache{TKey, TValue}"/> class.
36+
/// </summary>
37+
/// <param name="capacity"></param>
38+
public LruCache(int capacity)
39+
{
40+
_capacity = capacity;
41+
_linkedList = new LinkedList<KeyValuePair<TKey, TValue?>>();
42+
_map = new Dictionary<TKey, (LinkedListNode<KeyValuePair<TKey, TValue?>>, int)>();
43+
_syncLock = new object();
44+
}
45+
46+
/// <summary>
47+
/// Gets the value associated with the specified key.
48+
/// </summary>
49+
/// <param name="key">The key of the value to get.</param>
50+
/// <param name="value">When this method returns, contains the value associated with
51+
/// the specified key, if the key is found; otherwise, the default value for the type of
52+
/// the type of the <paramref name="value"/> parameter.</param>
53+
/// <returns><c>true</c> if the <see cref="LruCache{TKey, TValue}"/> contains an element
54+
/// with the specified key; otherwise, <c>false</c>.</returns>
55+
public bool TryGet(TKey key, out TValue? value)
56+
{
57+
lock (_syncLock)
58+
{
59+
if (_map.TryGetValue(key, out var mapValue))
60+
{
61+
var node = mapValue.Node;
62+
value = node.Value.Value;
63+
_linkedList.Remove(node);
64+
_linkedList.AddFirst(node);
65+
return true;
66+
}
67+
68+
value = default(TValue);
69+
return false;
70+
}
71+
}
72+
73+
/// <summary>
74+
/// Adds a key/value pair to the <see cref="LruCache{TKey, TValue}"/> if the key doesn't already exist, or updates a key/value
75+
/// pair in the <see cref="LruCache{TKey, TValue}"/> if the key does already exist.
76+
/// </summary>
77+
/// <param name="key"></param>
78+
/// <param name="val"></param>
79+
/// <param name="length"></param>
80+
public void AddOrUpdate(TKey key, TValue? val, int length)
81+
{
82+
lock (_syncLock)
83+
{
84+
if (_map.TryGetValue(key, out var existingValue))
85+
{
86+
// remove node - we will re-add a new node for this key at the head of the list, as the value may be different
87+
_linkedList.Remove(existingValue.Node);
88+
TotalLength -= _map[key].Length;
89+
}
90+
91+
// add new node
92+
var node = new LinkedListNode<KeyValuePair<TKey, TValue?>>(new KeyValuePair<TKey, TValue?>(key, val));
93+
_linkedList.AddFirst(node);
94+
_map[key] = (node, length);
95+
TotalLength += length;
96+
97+
if (_map.Count > _capacity)
98+
{
99+
// remove least recently used node
100+
LinkedListNode<KeyValuePair<TKey, TValue?>> last = _linkedList.Last!;
101+
_linkedList.RemoveLast();
102+
var toRemove = _map[last.Value.Key];
103+
_map.Remove(last.Value.Key);
104+
TotalLength -= toRemove.Length;
105+
}
106+
}
107+
}
108+
109+
/// <summary>
110+
/// Returns an enumerator that iterates through the <see cref="LruCache{TKey, TValue}"/>.
111+
/// </summary>
112+
/// <returns></returns>
113+
public IEnumerator<KeyValuePair<TKey, TValue>> GetEnumerator() => _linkedList.GetEnumerator();
114+
115+
IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
116+
}
117+
}

‎sdk/core/Azure.Core.Experimental/src/SchemaValidator/SchemaValidator.cs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
7+
namespace Azure.Core
8+
{
9+
/// <summary>
10+
/// This abstract class allows any available library to be used to generate schemas from .NET types and validate
11+
/// objects against schemas.
12+
/// </summary>
13+
/// <remarks>
14+
/// Defining both <see cref="GenerateSchema(Type)"/> and <see cref="TryValidate(object, Type, string, out IEnumerable{Exception})"/> is required. If you
15+
/// do not wish to validate, then evaluate all schemas as valid.
16+
/// </remarks>
17+
public abstract class SchemaValidator
18+
{
19+
/// <summary>
20+
/// Validates that <paramref name="data"/> is valid according to <paramref name="schemaDefinition"/>.
21+
/// </summary>
22+
/// <param name="data">The data to validate.</param>
23+
/// <param name="dataType">The type of data to validate.</param>
24+
/// <param name="schemaDefinition">The schema definition to validate against.</param>
25+
/// <param name="validationErrors">When this method returns, contains the validation errors if <paramref name="data"/> was invalid according to
26+
/// the <paramref name="schemaDefinition"/>.</param>
27+
/// <returns></returns>
28+
public abstract bool TryValidate(object data, Type dataType, string schemaDefinition, out IEnumerable<Exception> validationErrors);
29+
30+
/// <summary>
31+
/// Validates that <paramref name="data"/> is valid according to <paramref name="schemaDefinition"/>. If the object is not valid,
32+
/// this method throws an <see cref="AggregateException"/> containing all of the validation errors.
33+
/// </summary>
34+
/// <param name="data">The data to validate.</param>
35+
/// <param name="dataType">The type of data to validate.</param>
36+
/// <param name="schemaDefinition">The schema definition to validate against.</param>
37+
/// <exception cref="AggregateException"> <paramref name="data"/> is not valid according to the <paramref name="schemaDefinition"/>.</exception>
38+
public virtual void Validate(object data, Type dataType, string schemaDefinition)
39+
{
40+
if (!TryValidate(data, dataType, schemaDefinition, out var errors))
41+
{
42+
throw new AggregateException("The validate method determined the object was invalid according to the schema.", errors);
43+
}
44+
}
45+
46+
/// <summary>
47+
/// Generates a schema from <paramref name="dataType"/> and returns it as a string.
48+
/// </summary>
49+
/// <param name="dataType">The type of the data to use when generating the schema.</param>
50+
/// <returns>The generated schema in string format.</returns>
51+
public abstract string GenerateSchema(Type dataType);
52+
}
53+
}

‎sdk/schemaregistry/Microsoft.Azure.Data.SchemaRegistry.ApacheAvro/tests/LruCacheTests.cs‎ renamed to ‎sdk/core/Azure.Core.Experimental/tests/SchemaValidator/LruCacheTests.cs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1-
// Copyright (c) Microsoft Corporation. All rights reserved.
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
33

44
using NUnit.Framework;
5+
using Azure.Core;
56

6-
namespace Microsoft.Azure.Data.SchemaRegistry.ApacheAvro.Tests
7+
namespace Azure.Core.Experimental.Tests
78
{
89
public class LruCacheTests
910
{
@@ -65,4 +66,4 @@ public void CanUpdateExistingValue()
6566
Assert.AreEqual(10, val);
6667
}
6768
}
68-
}
69+
}

‎sdk/schemaregistry/Azure.Data.SchemaRegistry/Azure.Data.SchemaRegistry.sln‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11

22
Microsoft Visual Studio Solution File, Format Version 12.00
3-
# Visual Studio Version 16
4-
VisualStudioVersion = 16.0.29709.97
3+
# Visual Studio Version 17
4+
VisualStudioVersion = 17.5.33530.505
55
MinimumVisualStudioVersion = 10.0.40219.1
66
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Data.SchemaRegistry", "src\Azure.Data.SchemaRegistry.csproj", "{E33D09D9-D809-472C-82E6-6A26BDB86FC2}"
77
EndProject
@@ -27,10 +27,6 @@ Global
2727
{8052009B-2126-44A3-88CD-4F3B17894C64}.Debug|Any CPU.Build.0 = Debug|Any CPU
2828
{8052009B-2126-44A3-88CD-4F3B17894C64}.Release|Any CPU.ActiveCfg = Release|Any CPU
2929
{8052009B-2126-44A3-88CD-4F3B17894C64}.Release|Any CPU.Build.0 = Release|Any CPU
30-
{6A8336DC-AE5F-4955-95E3-BF5010A4CE5D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
31-
{6A8336DC-AE5F-4955-95E3-BF5010A4CE5D}.Debug|Any CPU.Build.0 = Debug|Any CPU
32-
{6A8336DC-AE5F-4955-95E3-BF5010A4CE5D}.Release|Any CPU.ActiveCfg = Release|Any CPU
33-
{6A8336DC-AE5F-4955-95E3-BF5010A4CE5D}.Release|Any CPU.Build.0 = Release|Any CPU
3430
EndGlobalSection
3531
GlobalSection(SolutionProperties) = preSolution
3632
HideSolutionNode = FALSE

‎sdk/schemaregistry/Azure.Data.SchemaRegistry/api/Azure.Data.SchemaRegistry.netstandard2.0.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,3 +68,28 @@ internal SchemaRegistrySchema() { }
6868
public Azure.Data.SchemaRegistry.SchemaProperties Properties { get { throw null; } }
6969
}
7070
}
71+
namespace Azure.Data.SchemaRegistry.Serialization
72+
{
73+
public partial class SchemaRegistrySerializer
74+
{
75+
protected SchemaRegistrySerializer() { }
76+
public SchemaRegistrySerializer(Azure.Data.SchemaRegistry.SchemaRegistryClient client, Azure.Core.SchemaValidator schemaValidator) { }
77+
public SchemaRegistrySerializer(Azure.Data.SchemaRegistry.SchemaRegistryClient client, Azure.Core.SchemaValidator schemaValidator, Azure.Data.SchemaRegistry.Serialization.SchemaRegistrySerializerOptions serializerOptions) { }
78+
public SchemaRegistrySerializer(Azure.Data.SchemaRegistry.SchemaRegistryClient client, string groupName, Azure.Core.SchemaValidator schemaValidator) { }
79+
public SchemaRegistrySerializer(Azure.Data.SchemaRegistry.SchemaRegistryClient client, string groupName, Azure.Core.SchemaValidator schemaValidator, Azure.Data.SchemaRegistry.Serialization.SchemaRegistrySerializerOptions serializerOptions) { }
80+
public object Deserialize(Azure.Messaging.MessageContent content, System.Type dataType, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
81+
public System.Threading.Tasks.ValueTask<object> DeserializeAsync(Azure.Messaging.MessageContent content, System.Type dataType, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
82+
public System.Threading.Tasks.ValueTask<TData> DeserializeAsync<TData>(Azure.Messaging.MessageContent content, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
83+
public TData Deserialize<TData>(Azure.Messaging.MessageContent content, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
84+
public Azure.Messaging.MessageContent Serialize(object data, System.Type dataType = null, System.Type messageType = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
85+
public System.Threading.Tasks.ValueTask<Azure.Messaging.MessageContent> SerializeAsync(object data, System.Type dataType = null, System.Type messageType = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
86+
public System.Threading.Tasks.ValueTask<TMessage> SerializeAsync<TMessage, TData>(TData data, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) where TMessage : Azure.Messaging.MessageContent, new() { throw null; }
87+
public TMessage Serialize<TMessage, TData>(TData data, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) where TMessage : Azure.Messaging.MessageContent, new() { throw null; }
88+
}
89+
public partial class SchemaRegistrySerializerOptions
90+
{
91+
public SchemaRegistrySerializerOptions() { }
92+
public Azure.Data.SchemaRegistry.SchemaFormat Format { get { throw null; } set { } }
93+
public Azure.Core.Serialization.ObjectSerializer Serializer { get { throw null; } set { } }
94+
}
95+
}

‎sdk/schemaregistry/Azure.Data.SchemaRegistry/assets.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
"AssetsRepo": "Azure/azure-sdk-assets",
33
"AssetsRepoPrefixPath": "net",
44
"TagPrefix": "net/schemaregistry/Azure.Data.SchemaRegistry",
5-
"Tag": "net/schemaregistry/Azure.Data.SchemaRegistry_7fedb15939"
5+
"Tag": "net/schemaregistry/Azure.Data.SchemaRegistry_e70c131e07"
66
}

0 commit comments

Comments
 (0)