Skip to content

Commit cf73109

Browse files
tg-msfttg-msft
authored
Fix Azure#16412 - overly permissive date parsing on facets (Azure#16693)
Fix Azure#16412 - overly permissive date parsing on facets
1 parent 7094b09 commit cf73109

File tree

9 files changed

+1754
-3
lines changed

9 files changed

+1754
-3
lines changed

‎sdk/search/Azure.Search.Documents/CHANGELOG.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,12 @@
66

77
- Fixed issue calling `SearchIndexClient.GetIndexNames` that threw an exception ([#15590](https://github.com/Azure/azure-sdk-for-net/issues/15590))
88
- Fixed issue where `ScoringProfile.FunctionAggregation` did not correctly handle null values ([#16570](https://github.com/Azure/azure-sdk-for-net/issues/16570))
9+
- Fixed overly permissive date parsing on facets ([#16412](https://github.com/Azure/azure-sdk-for-net/issues/16412))
910

1011
### Added
1112

1213
- Added `EncryptionKey` to `SearchIndexer`, `SearchIndexerDataSourceConnection`, and `SearchIndexerSkillset`.
13-
- Add configuration options to tune the performance of `SearchIndexingBufferedSender<T>`.
14+
- Added configuration options to tune the performance of `SearchIndexingBufferedSender<T>`.
1415

1516
## 11.2.0-beta.1 (2020-10-09)
1617

‎sdk/search/Azure.Search.Documents/src/Serialization/JsonSerialization.cs‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using System;
55
using System.Collections.Generic;
66
using System.Diagnostics;
7+
using System.Globalization;
78
using System.IO;
89
using System.Text;
910
using System.Text.Json;
@@ -23,6 +24,36 @@ namespace Azure.Search.Documents
2324
/// </summary>
2425
internal static class JsonSerialization
2526
{
27+
/// <summary>
28+
/// We serialize dates with the roundtrip format.
29+
/// </summary>
30+
private const string DateTimeOutputFormat = "o";
31+
32+
/// <summary>
33+
/// We parse dates using variations of the roundtrip format with
34+
/// different sub-second precision.
35+
/// </summary>
36+
private const string DateTimeInputFormatPrefix = "yyyy'-'MM'-'dd'T'HH':'mm':'ss";
37+
private static readonly string[] s_dateTimeInputFormats = new[]
38+
{
39+
DateTimeInputFormatPrefix + "zzz",
40+
DateTimeInputFormatPrefix + "K",
41+
DateTimeInputFormatPrefix + "'.'fzzz",
42+
DateTimeInputFormatPrefix + "'.'fK",
43+
DateTimeInputFormatPrefix + "'.'ffzzz",
44+
DateTimeInputFormatPrefix + "'.'ffK",
45+
DateTimeInputFormatPrefix + "'.'fffzzz",
46+
DateTimeInputFormatPrefix + "'.'fffK",
47+
DateTimeInputFormatPrefix + "'.'ffffzzz",
48+
DateTimeInputFormatPrefix + "'.'ffffK",
49+
DateTimeInputFormatPrefix + "'.'fffffzzz",
50+
DateTimeInputFormatPrefix + "'.'fffffK",
51+
DateTimeInputFormatPrefix + "'.'ffffffzzz",
52+
DateTimeInputFormatPrefix + "'.'ffffffK",
53+
DateTimeInputFormatPrefix + "'.'fffffffzzz",
54+
DateTimeInputFormatPrefix + "'.'fffffffK"
55+
};
56+
2657
/// <summary>
2758
/// Default JsonSerializerOptions to use.
2859
/// </summary>
@@ -97,7 +128,7 @@ public static string Date(DateTime value, IFormatProvider formatProvider) =>
97128
/// <param name="formatProvider">Format Provider.</param>
98129
/// <returns>OData string.</returns>
99130
public static string Date(DateTimeOffset value, IFormatProvider formatProvider) =>
100-
value.ToString("o", formatProvider);
131+
value.ToString(DateTimeOutputFormat, formatProvider);
101132

102133
/// <summary>
103134
/// Get a stream representation of a JsonElement. This is an
@@ -129,7 +160,12 @@ public static object GetSearchObject(this JsonElement element)
129160
Constants.InfValue => double.PositiveInfinity,
130161
Constants.NegativeInfValue => double.NegativeInfinity,
131162
string text =>
132-
DateTimeOffset.TryParse(text, out DateTimeOffset date) ?
163+
DateTimeOffset.TryParseExact(
164+
text,
165+
s_dateTimeInputFormats,
166+
CultureInfo.InvariantCulture,
167+
DateTimeStyles.RoundtripKind,
168+
out DateTimeOffset date) ?
133169
(object)date :
134170
(object)text
135171
};

‎sdk/search/Azure.Search.Documents/tests/DocumentOperations/SearchTests.cs‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
using System;
55
using System.Collections.Generic;
6+
using System.Globalization;
67
using System.Linq;
78
using System.Text.Json;
89
using System.Threading.Tasks;
@@ -11,6 +12,7 @@
1112
using Azure.Core.GeoJson;
1213
#endif
1314
using Azure.Core.TestFramework;
15+
using Azure.Search.Documents.Indexes;
1416
using Azure.Search.Documents.Models;
1517
using NUnit.Framework;
1618

@@ -663,6 +665,133 @@ await AssertKeysContains(
663665
Assert.AreEqual(4, second.Count);
664666
}
665667

668+
public class FacetKeyValuePair
669+
{
670+
public FacetKeyValuePair() { }
671+
public FacetKeyValuePair(string key, string value) { Key = key; Value = value; }
672+
673+
[SimpleField(IsKey = true)]
674+
public string Key { get; set; }
675+
676+
[SimpleField(IsFacetable = true)]
677+
public string Value { get; set; }
678+
}
679+
680+
[Test]
681+
public async Task FacetsArentAutomaticallyParsed()
682+
{
683+
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync<FacetKeyValuePair>(this);
684+
SearchClient client = resources.GetSearchClient();
685+
await client.UploadDocumentsAsync(
686+
new[]
687+
{
688+
new FacetKeyValuePair("1", "9-6"),
689+
new FacetKeyValuePair("2", "9.6"),
690+
new FacetKeyValuePair("3", "9'6\""),
691+
});
692+
await resources.WaitForIndexingAsync();
693+
694+
Response<SearchResults<FacetKeyValuePair>> response =
695+
await resources.GetQueryClient().SearchAsync<FacetKeyValuePair>(
696+
null,
697+
new SearchOptions { Facets = new[] { "Value" } });
698+
699+
Assert.IsNotNull(response.Value.Facets);
700+
AssertFacetsEqual(
701+
GetFacetsForField(response.Value.Facets, "Value", 3),
702+
MakeValueFacet(1, "9'6\""),
703+
MakeValueFacet(1, "9-6"),
704+
MakeValueFacet(1, "9.6"));
705+
706+
// Check strongly typed value facets
707+
ICollection<FacetResult> facets = GetFacetsForField(response.Value.Facets, "Value", 3);
708+
ValueFacetResult<string> first = facets.ElementAt(0).AsValueFacetResult<string>();
709+
Assert.AreEqual("9'6\"", first.Value);
710+
Assert.AreEqual(1, first.Count);
711+
ValueFacetResult<string> second = facets.ElementAt(1).AsValueFacetResult<string>();
712+
Assert.AreEqual("9-6", second.Value);
713+
Assert.AreEqual(1, second.Count);
714+
ValueFacetResult<string> third = facets.ElementAt(2).AsValueFacetResult<string>();
715+
Assert.AreEqual("9.6", third.Value);
716+
Assert.AreEqual(1, third.Count);
717+
}
718+
719+
[Test]
720+
public async Task FacetDateTimesRoundtrip()
721+
{
722+
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync<FacetKeyValuePair>(this);
723+
SearchClient client = resources.GetSearchClient();
724+
DateTimeOffset now = Recording.Now;
725+
726+
// Use valid dates
727+
string prefix = "yyyy'-'MM'-'dd'T'HH':'mm':'ss";
728+
FacetKeyValuePair[] data =
729+
new[]
730+
{
731+
new FacetKeyValuePair("1", now.ToString(prefix + "zzz", CultureInfo.InvariantCulture)),
732+
new FacetKeyValuePair("2", now.ToString(prefix + "K", CultureInfo.InvariantCulture)),
733+
new FacetKeyValuePair("3", now.ToString(prefix + "'.'fzzz", CultureInfo.InvariantCulture)),
734+
new FacetKeyValuePair("4", now.ToString(prefix + "'.'fK", CultureInfo.InvariantCulture)),
735+
new FacetKeyValuePair("5", now.ToString(prefix + "'.'ffzzz", CultureInfo.InvariantCulture)),
736+
new FacetKeyValuePair("6", now.ToString(prefix + "'.'ffK", CultureInfo.InvariantCulture)),
737+
new FacetKeyValuePair("7", now.ToString(prefix + "'.'fffzzz", CultureInfo.InvariantCulture)),
738+
new FacetKeyValuePair("8", now.ToString(prefix + "'.'fffK", CultureInfo.InvariantCulture)),
739+
new FacetKeyValuePair("9", now.ToString(prefix + "'.'ffffzzz", CultureInfo.InvariantCulture)),
740+
new FacetKeyValuePair("10", now.ToString(prefix + "'.'ffffK", CultureInfo.InvariantCulture)),
741+
new FacetKeyValuePair("11", now.ToString(prefix + "'.'fffffzzz", CultureInfo.InvariantCulture)),
742+
new FacetKeyValuePair("12", now.ToString(prefix + "'.'fffffK", CultureInfo.InvariantCulture)),
743+
new FacetKeyValuePair("13", now.ToString(prefix + "'.'ffffffzzz", CultureInfo.InvariantCulture)),
744+
new FacetKeyValuePair("14", now.ToString(prefix + "'.'ffffffK", CultureInfo.InvariantCulture)),
745+
new FacetKeyValuePair("15", now.ToString(prefix + "'.'fffffffzzz", CultureInfo.InvariantCulture)),
746+
new FacetKeyValuePair("16", now.ToString(prefix + "'.'fffffffK", CultureInfo.InvariantCulture))
747+
};
748+
await client.UploadDocumentsAsync(data);
749+
await resources.WaitForIndexingAsync();
750+
751+
Response<SearchResults<SearchDocument>> response =
752+
await resources.GetQueryClient().SearchAsync<SearchDocument>(
753+
null,
754+
new SearchOptions { Facets = new[] { "Value,count:" + data.Length } });
755+
foreach (FacetResult facet in response.Value.Facets["Value"])
756+
{
757+
Assert.IsInstanceOf(
758+
typeof(DateTimeOffset),
759+
facet.Value,
760+
$"Expected a DateTimeOffset, not {facet.Value} of type {facet.Value?.GetType().FullName}");
761+
}
762+
}
763+
764+
[Test]
765+
public async Task FacetDateTimesInvalid()
766+
{
767+
await using SearchResources resources = await SearchResources.CreateWithEmptyIndexAsync<FacetKeyValuePair>(this);
768+
SearchClient client = resources.GetSearchClient();
769+
DateTimeOffset now = Recording.Now;
770+
771+
// Use invalid dates
772+
FacetKeyValuePair[] data =
773+
new[]
774+
{
775+
new FacetKeyValuePair("1", now.ToString("yyyy'-'MM'-'dd'T'HH':'mm", CultureInfo.InvariantCulture)),
776+
new FacetKeyValuePair("2", now.ToString("yyyy'-'MM'-'dd", CultureInfo.InvariantCulture)),
777+
new FacetKeyValuePair("3", now.ToString("HH':'mm", CultureInfo.InvariantCulture))
778+
};
779+
await client.UploadDocumentsAsync(data);
780+
await resources.WaitForIndexingAsync();
781+
782+
Response<SearchResults<SearchDocument>> response =
783+
await resources.GetQueryClient().SearchAsync<SearchDocument>(
784+
null,
785+
new SearchOptions { Facets = new[] { "Value,count:" + data.Length } });
786+
foreach (FacetResult facet in response.Value.Facets["Value"])
787+
{
788+
Assert.IsInstanceOf(
789+
typeof(string),
790+
facet.Value,
791+
$"Expected a string, not {facet.Value} of type {facet.Value?.GetType().FullName}");
792+
}
793+
}
794+
666795
[Test]
667796
public async Task CanContinueStatic()
668797
{

0 commit comments

Comments
 (0)