[Search] Add support for lexical normalizers (#23690)

* Add support for lexical normaliers

* Remove copies of code in ctors

* Update test session records

Author: Mohit-Chakraborty

sdk/search/Azure.Search.Documents/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Release History
 
+## 11.4.0-beta.3 (2021-09-07)
+
+### Features Added
+- Support for [Lexical normalizers](https://docs.microsoft.com/azure/search/search-normalizers#normalizers) in [text analysers](https://docs.microsoft.com/rest/api/searchservice/test-analyzer) via `AnalyzeTextOptions`.
+
 ## 11.4.0-beta.2 (2021-08-10)
 
 ### Features Added

sdk/search/Azure.Search.Documents/api/Azure.Search.Documents.netstandard2.0.cs

@@ -324,9 +324,11 @@ public partial class AnalyzeTextOptions
     {
         public AnalyzeTextOptions(string text) { }
         public AnalyzeTextOptions(string text, Azure.Search.Documents.Indexes.Models.LexicalAnalyzerName analyzerName) { }
+        public AnalyzeTextOptions(string text, Azure.Search.Documents.Indexes.Models.LexicalNormalizerName normalizerName) { }
         public AnalyzeTextOptions(string text, Azure.Search.Documents.Indexes.Models.LexicalTokenizerName tokenizerName) { }
         public Azure.Search.Documents.Indexes.Models.LexicalAnalyzerName? AnalyzerName { get { throw null; } }
         public System.Collections.Generic.IList<string> CharFilters { get { throw null; } }
+        public Azure.Search.Documents.Indexes.Models.LexicalNormalizerName? NormalizerName { get { throw null; } }
         public string Text { get { throw null; } }
         public System.Collections.Generic.IList<Azure.Search.Documents.Indexes.Models.TokenFilterName> TokenFilters { get { throw null; } }
         public Azure.Search.Documents.Indexes.Models.LexicalTokenizerName? TokenizerName { get { throw null; } }

sdk/search/Azure.Search.Documents/src/Azure.Search.Documents.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <AssemblyTitle>Microsoft Azure.Search.Documents client library</AssemblyTitle>
-    <Version>11.4.0-beta.2</Version>
+    <Version>11.4.0-beta.3</Version>
     <!--The ApiCompatVersion is managed automatically and should not generally be modified manually.-->
     <ApiCompatVersion>11.3.0</ApiCompatVersion>
     <Description>

sdk/search/Azure.Search.Documents/src/Generated/Models/AnalyzeTextOptions.Serialization.cs

@@ -16,42 +16,43 @@ public partial class AnalyzeTextOptions
         /// <param name="text">Required text to break into tokens.</param>
         /// <param name="analyzerName">The name of the analyzer to use to break the given <paramref name="text"/>.</param>
         /// <exception cref="ArgumentNullException"><paramref name="text"/> is null.</exception>
-        public AnalyzeTextOptions(string text, LexicalAnalyzerName analyzerName)
-        {
-            Text = text ?? throw new ArgumentNullException(nameof(text));
-            AnalyzerName = analyzerName;
-
-            TokenFilters = new ChangeTrackingList<TokenFilterName>();
-            CharFilters = new ChangeTrackingList<string>();
-        }
+        public AnalyzeTextOptions(string text, LexicalAnalyzerName analyzerName) : this(text)
+            => AnalyzerName = analyzerName;
 
         /// <summary>
         /// Initializes a new instance of AnalyzeRequest.
         /// </summary>
         /// <param name="text">Required text to break into tokens.</param>
         /// <param name="tokenizerName">The name of the tokenizer to use to break the given <paramref name="text"/>.</param>
         /// <exception cref="ArgumentNullException"><paramref name="text"/> is null.</exception>
-        public AnalyzeTextOptions(string text, LexicalTokenizerName tokenizerName)
-        {
-            Text = text ?? throw new ArgumentNullException(nameof(text));
-            TokenizerName = tokenizerName;
+        public AnalyzeTextOptions(string text, LexicalTokenizerName tokenizerName) : this(text)
+            => TokenizerName = tokenizerName;
 
-            TokenFilters = new ChangeTrackingList<TokenFilterName>();
-            CharFilters = new ChangeTrackingList<string>();
-        }
+        /// <summary>
+        /// Initializes a new instance of AnalyzeRequest.
+        /// </summary>
+        /// <param name="text">Required text to break into tokens.</param>
+        /// <param name="normalizerName">The name of the tokenizer to use to break the given <paramref name="text"/>.</param>
+        /// <exception cref="ArgumentNullException"><paramref name="text"/> is null.</exception>
+        public AnalyzeTextOptions(string text, LexicalNormalizerName normalizerName) : this(text)
+            => NormalizerName = normalizerName;
 
-        /// <summary> The name of the analyzer to use to break the given text. If this parameter is not specified, you must specify a tokenizer instead. The tokenizer and analyzer parameters are mutually exclusive. </summary>
+        /// <summary> The name of the analyzer to use to break the given text. </summary>
         [CodeGenMember("Analyzer")]
         public LexicalAnalyzerName? AnalyzerName { get; }
 
-        /// <summary> The name of the tokenizer to use to break the given text. If this parameter is not specified, you must specify an analyzer instead. The tokenizer and analyzer parameters are mutually exclusive. </summary>
+        /// <summary> The name of the tokenizer to use to break the given text. </summary>
         [CodeGenMember("Tokenizer")]
         public LexicalTokenizerName? TokenizerName { get; }
 
-        /// <summary> An optional list of token filters to use when breaking the given text. This parameter can only be set when using the tokenizer parameter. </summary>
+        /// <summary> The name of the normalizer to use to normalize the given text. </summary>
+        [CodeGenMember("Normalizer")]
+        public LexicalNormalizerName? NormalizerName { get; }
+
+        /// <summary> An optional list of token filters to use when breaking the given text. </summary>
         public IList<TokenFilterName> TokenFilters { get; }
 
-        /// <summary> An optional list of character filters to use when breaking the given text. This parameter can only be set when using the tokenizer parameter. </summary>
+        /// <summary> An optional list of character filters to use when breaking the given text. </summary>
         public IList<string> CharFilters { get; }
     }
 }

sdk/search/Azure.Search.Documents/src/Indexes/Models/AnalyzeTextOptions.cs

@@ -13,8 +13,8 @@ copy them locally in `/sdk/search/generate.ps1` and reference them here.
 ```yaml
 title: SearchServiceClient
 input-file:
-- https://raw.githubusercontent.com/Azure/azure-rest-api-specs/d2183715d380084ff04313a73c8803d042fe91b9/specification/search/data-plane/Azure.Search/preview/2021-04-30-Preview/searchindex.json
-- https://raw.githubusercontent.com/Azure/azure-rest-api-specs/d2183715d380084ff04313a73c8803d042fe91b9/specification/search/data-plane/Azure.Search/preview/2021-04-30-Preview/searchservice.json
+- https://raw.githubusercontent.com/Azure/azure-rest-api-specs/c99fbb96d7993daec8135a40681d9d807e3f5751/specification/search/data-plane/Azure.Search/preview/2021-04-30-Preview/searchindex.json
+- https://raw.githubusercontent.com/Azure/azure-rest-api-specs/c99fbb96d7993daec8135a40681d9d807e3f5751/specification/search/data-plane/Azure.Search/preview/2021-04-30-Preview/searchservice.json
 ```
 
 ## Release hacks

sdk/search/Azure.Search.Documents/src/autorest.md

@@ -15,6 +15,7 @@
 
 namespace Azure.Search.Documents.Tests
 {
+    [ClientTestFixture(SearchClientOptions.ServiceVersion.V2020_06_30, SearchClientOptions.ServiceVersion.V2021_04_30_Preview)]
     public class SearchIndexClientTests : SearchTestBase
     {
         public SearchIndexClientTests(bool async, SearchClientOptions.ServiceVersion serviceVersion)
@@ -446,6 +447,31 @@ public async Task AnalyzeText()
             Assert.AreEqual(new[] { "The", "quick", "brown", "fox", "jumped", "over", "the", "lazy", "dog." }, tokens.Select(t => t.Token));
         }
 
+        [Test]
+        [ServiceVersion(Min = SearchClientOptions.ServiceVersion.V2021_04_30_Preview)]
+        public async Task AnalyzeTextWithNormalizer()
+        {
+            await using SearchResources resources = await SearchResources.GetSharedHotelsIndexAsync(this);
+
+            SearchIndexClient client = resources.GetIndexClient();
+
+            AnalyzeTextOptions request = new("I dARe YoU tO reAd It IN A nORmAl vOiCE.", LexicalNormalizerName.Lowercase);
+
+            Response<IReadOnlyList<AnalyzedTokenInfo>> result = await client.AnalyzeTextAsync(resources.IndexName, request);
+            IReadOnlyList<AnalyzedTokenInfo> tokens = result.Value;
+
+            Assert.AreEqual(1, tokens.Count);
+            Assert.AreEqual("i dare you to read it in a normal voice.", tokens[0].Token);
+
+            request = new("Item ① in my ⑽ point rant is that 75⁰F is uncomfortably warm.", LexicalNormalizerName.AsciiFolding);
+
+            result = await client.AnalyzeTextAsync(resources.IndexName, request);
+            tokens = result.Value;
+
+            Assert.AreEqual(1, tokens.Count);
+            Assert.AreEqual("Item 1 in my (10) point rant is that 750F is uncomfortably warm.", tokens[0].Token);
+        }
+
         [Test]
         public async Task SetScoringProfile()
         {