Skip to content

Commit efe535b

Browse files
pakrympakrym
authored
Add support for timespans in log queries (Azure#20528)
1 parent de1abf5 commit efe535b

24 files changed

+657
-181
lines changed

‎sdk/monitor/Azure.Monitory.Query/api/Azure.Monitory.Query.netstandard2.0.cs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ namespace Azure.Monitory.Query
33
public partial class LogsBatchQuery
44
{
55
protected LogsBatchQuery() { }
6-
public virtual string AddQuery(string workspaceId, string query) { throw null; }
6+
public virtual string AddQuery(string workspaceId, string query, System.TimeSpan? timeSpan = default(System.TimeSpan?)) { throw null; }
77
public virtual Azure.Response<Azure.Monitory.Query.Models.LogsBatchQueryResult> Submit(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
88
public virtual System.Threading.Tasks.Task<Azure.Response<Azure.Monitory.Query.Models.LogsBatchQueryResult>> SubmitAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
99
}
@@ -13,10 +13,10 @@ protected LogsClient() { }
1313
public LogsClient(Azure.Core.TokenCredential credential) { }
1414
public LogsClient(Azure.Core.TokenCredential credential, Azure.Monitory.Query.LogsClientOptions options) { }
1515
public virtual Azure.Monitory.Query.LogsBatchQuery CreateBatchQuery() { throw null; }
16-
public virtual Azure.Response<Azure.Monitory.Query.Models.LogsQueryResult> Query(string workspaceId, string query, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
17-
public virtual System.Threading.Tasks.Task<Azure.Response<Azure.Monitory.Query.Models.LogsQueryResult>> QueryAsync(string workspaceId, string query, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
18-
public virtual System.Threading.Tasks.Task<Azure.Response<System.Collections.Generic.IReadOnlyList<T>>> QueryAsync<T>(string workspaceId, string query, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
19-
public virtual Azure.Response<System.Collections.Generic.IReadOnlyList<T>> Query<T>(string workspaceId, string query, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
16+
public virtual Azure.Response<Azure.Monitory.Query.Models.LogsQueryResult> Query(string workspaceId, string query, System.TimeSpan? timeSpan = default(System.TimeSpan?), System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
17+
public virtual System.Threading.Tasks.Task<Azure.Response<Azure.Monitory.Query.Models.LogsQueryResult>> QueryAsync(string workspaceId, string query, System.TimeSpan? timeSpan = default(System.TimeSpan?), System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
18+
public virtual System.Threading.Tasks.Task<Azure.Response<System.Collections.Generic.IReadOnlyList<T>>> QueryAsync<T>(string workspaceId, string query, System.TimeSpan? timeSpan = default(System.TimeSpan?), System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
19+
public virtual Azure.Response<System.Collections.Generic.IReadOnlyList<T>> Query<T>(string workspaceId, string query, System.TimeSpan? timeSpan = default(System.TimeSpan?), System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
2020
}
2121
public partial class LogsClientOptions : Azure.Core.ClientOptions
2222
{

‎sdk/monitor/Azure.Monitory.Query/src/LogsClient.cs‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,6 @@
33

44
using System;
55
using System.Collections.Generic;
6-
using System.Globalization;
7-
using System.Linq;
8-
using System.Reflection;
96
using System.Threading;
107
using System.Threading.Tasks;
118
using Azure.Core;
@@ -39,27 +36,27 @@ protected LogsClient()
3936
{
4037
}
4138

42-
public virtual Response<IReadOnlyList<T>> Query<T>(string workspaceId, string query, CancellationToken cancellationToken = default)
39+
public virtual Response<IReadOnlyList<T>> Query<T>(string workspaceId, string query, TimeSpan? timeSpan = null, CancellationToken cancellationToken = default)
4340
{
44-
Response<LogsQueryResult> response = Query(workspaceId, query, cancellationToken);
41+
Response<LogsQueryResult> response = Query(workspaceId, query, timeSpan, cancellationToken);
4542

4643
return Response.FromValue(RowBinder.BindResults<T>(response), response.GetRawResponse());
4744
}
4845

49-
public virtual async Task<Response<IReadOnlyList<T>>> QueryAsync<T>(string workspaceId, string query, CancellationToken cancellationToken = default)
46+
public virtual async Task<Response<IReadOnlyList<T>>> QueryAsync<T>(string workspaceId, string query, TimeSpan? timeSpan = null, CancellationToken cancellationToken = default)
5047
{
51-
Response<LogsQueryResult> response = await QueryAsync(workspaceId, query, cancellationToken).ConfigureAwait(false);
48+
Response<LogsQueryResult> response = await QueryAsync(workspaceId, query, timeSpan, cancellationToken).ConfigureAwait(false);
5249

5350
return Response.FromValue(RowBinder.BindResults<T>(response), response.GetRawResponse());
5451
}
5552

56-
public virtual Response<LogsQueryResult> Query(string workspaceId, string query, CancellationToken cancellationToken = default)
53+
public virtual Response<LogsQueryResult> Query(string workspaceId, string query, TimeSpan? timeSpan = null, CancellationToken cancellationToken = default)
5754
{
5855
using DiagnosticScope scope = _clientDiagnostics.CreateScope($"{nameof(LogsClient)}.{nameof(Query)}");
5956
scope.Start();
6057
try
6158
{
62-
return _queryClient.Execute(workspaceId, new QueryBody(query), null, cancellationToken);
59+
return _queryClient.Execute(workspaceId, CreateQueryBody(query, timeSpan), null, cancellationToken);
6360
}
6461
catch (Exception e)
6562
{
@@ -68,13 +65,13 @@ public virtual Response<LogsQueryResult> Query(string workspaceId, string query,
6865
}
6966
}
7067

71-
public virtual async Task<Response<LogsQueryResult>> QueryAsync(string workspaceId, string query, CancellationToken cancellationToken = default)
68+
public virtual async Task<Response<LogsQueryResult>> QueryAsync(string workspaceId, string query, TimeSpan? timeSpan = null, CancellationToken cancellationToken = default)
7269
{
7370
using DiagnosticScope scope = _clientDiagnostics.CreateScope($"{nameof(LogsClient)}.{nameof(Query)}");
7471
scope.Start();
7572
try
7673
{
77-
return await _queryClient.ExecuteAsync(workspaceId, new QueryBody(query), null, cancellationToken).ConfigureAwait(false);
74+
return await _queryClient.ExecuteAsync(workspaceId, CreateQueryBody(query, timeSpan), null, cancellationToken).ConfigureAwait(false);
7875
}
7976
catch (Exception e)
8077
{
@@ -87,5 +84,16 @@ public virtual LogsBatchQuery CreateBatchQuery()
8784
{
8885
return new LogsBatchQuery(_clientDiagnostics, _queryClient);
8986
}
87+
88+
internal static QueryBody CreateQueryBody(string query, TimeSpan? timeSpan)
89+
{
90+
var queryBody = new QueryBody(query);
91+
if (timeSpan != null)
92+
{
93+
queryBody.Timespan = TypeFormatters.ToString(timeSpan.Value, "P");
94+
}
95+
96+
return queryBody;
97+
}
9098
}
9199
}

‎sdk/monitor/Azure.Monitory.Query/src/Models/LogsBatchQuery.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,14 @@ protected LogsBatchQuery()
2828
{
2929
}
3030

31-
public virtual string AddQuery(string workspaceId, string query)
31+
public virtual string AddQuery(string workspaceId, string query, TimeSpan? timeSpan = null)
3232
{
3333
var id = _counter.ToString("G", CultureInfo.InvariantCulture);
3434
_counter++;
3535
_batch.Requests.Add(new LogQueryRequest()
3636
{
3737
Id = id,
38-
Body = new QueryBody(query),
38+
Body = LogsClient.CreateQueryBody(query, timeSpan),
3939
Workspace = workspaceId
4040
});
4141
return id;

‎sdk/monitor/Azure.Monitory.Query/tests/LogSenderClient.cs‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
using System;
55
using System.Collections.Generic;
6+
using System.IO;
67
using System.Security.Cryptography;
78
using System.Text;
89
using System.Text.Json;
@@ -30,13 +31,24 @@ public LogSenderClient(string workspaceId, string ingestEndpointSuffix, string s
3031

3132
public async Task<Response> SendAsync(string tableName, IEnumerable<IDictionary<string, object>> values)
3233
{
33-
var data = JsonSerializer.Serialize(values);
34+
byte[] data;
35+
using (var stream = new MemoryStream())
36+
{
37+
using (var writer = new Utf8JsonWriter(stream))
38+
{
39+
writer.WriteObjectValue(values);
40+
}
41+
42+
data = stream.ToArray();
43+
}
44+
3445
var request = _pipeline.CreateRequest();
3546
request.Uri.Reset(new Uri($"https://{_workspaceId}.{_ingestEndpointSuffix}/api/logs?api-version=2016-04-01"));
3647
request.Method = RequestMethod.Post;
3748
request.Headers.SetValue("Content-Type", "application/json");
3849
request.Headers.SetValue("Log-Type", tableName);
39-
request.Content = data;
50+
request.Headers.SetValue("time-generated-field", "EventTimeGenerated");
51+
request.Content = RequestContent.Create(data);
4052

4153
var response = await _pipeline.SendRequestAsync(request, default);
4254
if (response.Status != 200)

‎sdk/monitor/Azure.Monitory.Query/tests/LogsQueryClientClientLiveTests.cs‎

Lines changed: 55 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
33

4+
using System;
45
using System.Collections.Generic;
6+
using System.Globalization;
57
using System.Linq;
68
using System.Threading.Tasks;
79
using Azure.Core.TestFramework;
@@ -21,7 +23,7 @@ public LogsQueryClientClientLiveTests(bool isAsync) : base(isAsync)
2123
[SetUp]
2224
public async Task SetUp()
2325
{
24-
_logsTestData = new LogsTestData(TestEnvironment);
26+
_logsTestData = new LogsTestData(this);
2527
await _logsTestData.InitializeAsync();
2628
}
2729

@@ -39,7 +41,7 @@ public async Task CanQuery()
3941
var client = CreateClient();
4042

4143
var results = await client.QueryAsync(TestEnvironment.WorkspaceId,
42-
$"{LogsTestData.TableAName} |" +
44+
$"{_logsTestData.TableAName} |" +
4345
$"project {LogsTestData.StringColumnName}, {LogsTestData.IntColumnName}, {LogsTestData.BoolColumnName}, {LogsTestData.FloatColumnName} |" +
4446
$"order by {LogsTestData.StringColumnName} asc");
4547

@@ -65,7 +67,7 @@ public async Task CanQueryIntoPrimitiveString()
6567
var client = CreateClient();
6668

6769
var results = await client.QueryAsync<string>(TestEnvironment.WorkspaceId,
68-
$"{LogsTestData.TableAName} | project {LogsTestData.StringColumnName} | order by {LogsTestData.StringColumnName} asc");
70+
$"{_logsTestData.TableAName} | project {LogsTestData.StringColumnName} | order by {LogsTestData.StringColumnName} asc");
6971

7072
CollectionAssert.AreEqual(new[] {"a","b","c"}, results.Value);
7173
}
@@ -75,9 +77,9 @@ public async Task CanQueryIntoPrimitiveInt()
7577
{
7678
var client = CreateClient();
7779

78-
var results = await client.QueryAsync<int>(TestEnvironment.WorkspaceId, $"{LogsTestData.TableAName} | count");
80+
var results = await client.QueryAsync<int>(TestEnvironment.WorkspaceId, $"{_logsTestData.TableAName} | count");
7981

80-
Assert.AreEqual(LogsTestData.TableA.Count, results.Value[0]);
82+
Assert.AreEqual(_logsTestData.TableA.Count, results.Value[0]);
8183
}
8284

8385
[RecordedTest]
@@ -86,7 +88,7 @@ public async Task CanQueryIntoClass()
8688
var client = CreateClient();
8789

8890
var results = await client.QueryAsync<TestModel>(TestEnvironment.WorkspaceId,
89-
$"{LogsTestData.TableAName} |" +
91+
$"{_logsTestData.TableAName} |" +
9092
$"project-rename Name = {LogsTestData.StringColumnName}, Age = {LogsTestData.IntColumnName} |" +
9193
$"order by Name asc");
9294

@@ -104,7 +106,7 @@ public async Task CanQueryIntoDictionary()
104106
var client = CreateClient();
105107

106108
var results = await client.QueryAsync<Dictionary<string, object>>(TestEnvironment.WorkspaceId,
107-
$"{LogsTestData.TableAName} |" +
109+
$"{_logsTestData.TableAName} |" +
108110
$"project-rename Name = {LogsTestData.StringColumnName}, Age = {LogsTestData.IntColumnName} |" +
109111
$"project Name, Age |" +
110112
$"order by Name asc");
@@ -123,7 +125,7 @@ public async Task CanQueryIntoIDictionary()
123125
var client = CreateClient();
124126

125127
var results = await client.QueryAsync<IDictionary<string, object>>(TestEnvironment.WorkspaceId,
126-
$"{LogsTestData.TableAName} |" +
128+
$"{_logsTestData.TableAName} |" +
127129
$"project-rename Name = {LogsTestData.StringColumnName}, Age = {LogsTestData.IntColumnName} |" +
128130
$"project Name, Age |" +
129131
$"order by Name asc");
@@ -152,6 +154,51 @@ public async Task CanQueryBatch()
152154
CollectionAssert.IsNotEmpty(result2.Tables[0].Columns);
153155
}
154156

157+
[RecordedTest]
158+
public async Task CanQueryWithTimespan()
159+
{
160+
// Get the time of the second event and add a bit of buffer to it (events are 2d apart)
161+
var minOffset = (DateTimeOffset)_logsTestData.TableA[1][LogsTestData.TimeGeneratedColumnNameSent];
162+
var timespan = Recording.UtcNow - minOffset;
163+
timespan = timespan.Add(TimeSpan.FromDays(1));
164+
165+
var client = CreateClient();
166+
var results = await client.QueryAsync<string>(
167+
TestEnvironment.WorkspaceId,
168+
$"{_logsTestData.TableAName} | project {LogsTestData.TimeGeneratedColumnName}",
169+
timespan);
170+
171+
// We should get the second and the third events
172+
Assert.AreEqual(2, results.Value.Count);
173+
// TODO: Switch to querying DateTimeOffset
174+
Assert.True(results.Value.All(r => DateTimeOffset.Parse(r, null, DateTimeStyles.AssumeUniversal) >= minOffset));
175+
}
176+
177+
[RecordedTest]
178+
public async Task CanQueryBatchWithTimespan()
179+
{
180+
// Get the time of the second event and add a bit of buffer to it (events are 2d apart)
181+
var minOffset = (DateTimeOffset)_logsTestData.TableA[1][LogsTestData.TimeGeneratedColumnNameSent];
182+
var timespan = Recording.UtcNow - minOffset;
183+
timespan = timespan.Add(TimeSpan.FromDays(1));
184+
185+
var client = CreateClient();
186+
LogsBatchQuery batch = InstrumentClient(client.CreateBatchQuery());
187+
string id1 = batch.AddQuery(TestEnvironment.WorkspaceId, $"{_logsTestData.TableAName} | project {LogsTestData.TimeGeneratedColumnName}");
188+
string id2 = batch.AddQuery(TestEnvironment.WorkspaceId, $"{_logsTestData.TableAName} | project {LogsTestData.TimeGeneratedColumnName}", timespan);
189+
Response<LogsBatchQueryResult> response = await batch.SubmitAsync();
190+
191+
var result1 = response.Value.GetResult<string>(id1);
192+
var result2 = response.Value.GetResult<string>(id2);
193+
194+
// All rows
195+
Assert.AreEqual(3, result1.Count);
196+
// Filtered by the timestamp
197+
Assert.AreEqual(2, result2.Count);
198+
// TODO: Switch to querying DateTimeOffset
199+
Assert.True(result2.All(r => DateTimeOffset.Parse(r, null, DateTimeStyles.AssumeUniversal) >= minOffset));
200+
}
201+
155202
private record TestModel
156203
{
157204
public string Name { get; set; }

‎sdk/monitor/Azure.Monitory.Query/tests/LogsTestData.cs‎

Lines changed: 45 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@ namespace Azure.Template.Tests
1515
public class LogsTestData
1616
{
1717
private static readonly string DataVersion = "1";
18+
// The data retention time is 31 day by-default so we need to make sure the data we posted is still
19+
// being retained.
20+
// Make the windows start the monday of a previous week.
21+
private DateTimeOffset RetentionWindowStart;
1822

1923
public static string IntColumnNameSent = "IntColumn";
2024
public static string IntColumnName = IntColumnNameSent + "_d";
@@ -28,22 +32,52 @@ public class LogsTestData
2832
public static string FloatColumnNameSent = "FloatColumn";
2933
public static string FloatColumnName = FloatColumnNameSent + "_d";
3034

31-
public static readonly List<Dictionary<string, object>> TableA = new()
32-
{
33-
new() { { IntColumnNameSent, 1}, { StringColumnNameSent, "a"}, { BoolColumnNameSent, false}, { FloatColumnNameSent, 0f } },
34-
new() { { IntColumnNameSent, 3}, { StringColumnNameSent, "b"}, { BoolColumnNameSent, true}, { FloatColumnNameSent, 1.2f } },
35-
new() { { IntColumnNameSent, 1}, { StringColumnNameSent, "c"}, { BoolColumnNameSent, false}, { FloatColumnNameSent, 1.1f } },
36-
};
35+
public static string TimeGeneratedColumnNameSent = "EventTimeGenerated";
36+
public static string TimeGeneratedColumnName = "TimeGenerated";
37+
38+
public readonly List<Dictionary<string, object>> TableA;
3739

38-
private static string TableANameSent = nameof(TableA) + DataVersion;
39-
public static string TableAName = TableANameSent + "_CL";
40+
private string TableANameSent => nameof(TableA) + DataVersion + "_" + RetentionWindowStart.DayOfYear;
41+
public string TableAName => TableANameSent + "_CL";
4042

4143
private readonly MonitorQueryClientTestEnvironment _testEnvironment;
42-
private bool _initialized;
44+
private static bool _initialized;
4345

44-
public LogsTestData(MonitorQueryClientTestEnvironment testEnvironment)
46+
public LogsTestData(RecordedTestBase<MonitorQueryClientTestEnvironment> test)
4547
{
46-
_testEnvironment = testEnvironment;
48+
_testEnvironment = test.TestEnvironment;
49+
50+
// Make sure we don't need to re-record every week
51+
var recordingUtcNow = DateTime.SpecifyKind(test.Recording.UtcNow.Date, DateTimeKind.Utc);
52+
RetentionWindowStart = recordingUtcNow.AddDays(DayOfWeek.Monday - recordingUtcNow.DayOfWeek - 7);
53+
54+
TableA = new()
55+
{
56+
new()
57+
{
58+
{ IntColumnNameSent, 1},
59+
{ StringColumnNameSent, "a"},
60+
{ BoolColumnNameSent, false},
61+
{ FloatColumnNameSent, 0f },
62+
{ TimeGeneratedColumnNameSent, RetentionWindowStart }
63+
},
64+
new()
65+
{
66+
{ IntColumnNameSent, 3},
67+
{ StringColumnNameSent, "b"},
68+
{ BoolColumnNameSent, true},
69+
{ FloatColumnNameSent, 1.2f },
70+
{ TimeGeneratedColumnNameSent, RetentionWindowStart.AddDays(2) }
71+
},
72+
new()
73+
{
74+
{ IntColumnNameSent, 1},
75+
{ StringColumnNameSent, "c"},
76+
{ BoolColumnNameSent, false},
77+
{ FloatColumnNameSent, 1.1f },
78+
{ TimeGeneratedColumnNameSent, RetentionWindowStart.AddDays(5) }
79+
},
80+
};
4781
}
4882

4983
public async Task InitializeAsync()

0 commit comments

Comments
 (0)