Introduce an exponential retry for service timing issue causing flakey Live tests (Azure#15159)

see https://dev.azure.com/azure-sdk/internal/_build/results?buildId=537514&view=results

Author: christothes

sdk/tables/Azure.Data.Tables/tests/TableServiceClientLiveTests.cs

@@ -245,17 +245,13 @@ public async Task GetPropertiesReturnsProperties()
 
             await service.SetPropertiesAsync(responseToChange).ConfigureAwait(false);
 
-            // Wait 20 sec if on Live mode to ensure properties are updated in the service
-            // Minimum time: Sync - 20 sec; Async - 12 sec
-
-            if (Mode != RecordedTestMode.Playback)
-            {
-                await Task.Delay(20000);
-            }
-
             // Get configured properties
+            // A delay is required to ensure properties are updated in the service
 
-            TableServiceProperties changedResponse = await service.GetPropertiesAsync().ConfigureAwait(false);
+            TableServiceProperties changedResponse = await RetryUntilExpectedResponse(
+                async () => await service.GetPropertiesAsync().ConfigureAwait(false),
+                result => result.Value.Logging.Read == responseToChange.Logging.Read,
+                15000).ConfigureAwait(false);
 
             // Test each property
 

sdk/tables/Azure.Data.Tables/tests/TableServiceLiveTestsBase.cs

@@ -262,6 +262,27 @@ protected async Task<TResult> CosmosThrottleWrapper<TResult>(Func<Task<TResult>>
             }
         }
 
+        protected async Task<TResult> RetryUntilExpectedResponse<TResult>(Func<Task<TResult>> action, Func<TResult, bool> equalityAction, int initialDelay)
+        {
+            int retryCount = 0;
+            int delay = initialDelay;
+            while (true)
+            {
+                var actual = await action().ConfigureAwait(false);
+
+                if (++retryCount > 3 || equalityAction(actual))
+                {
+                    return actual;
+                }
+                // Disable retry throttling in Playback mode.
+                if (Mode != RecordedTestMode.Playback)
+                {
+                    await Task.Delay(delay);
+                    delay *= 2;
+                }
+            }
+        }
+
         protected async Task CreateTestEntities<T>(List<T> entitiesToCreate) where T : class, ITableEntity, new()
         {
             foreach (var entity in entitiesToCreate)