Exclude test data from functions endpoint response (#10943) (#10981)

kshyju · web-flow · commit 2485b8cac9b7 · 2025-04-02T11:33:09.000-07:00
diff --git a/release_notes.md b/release_notes.md
@@ -2,4 +2,6 @@
 
 <!-- Please add your release notes in the following format:
 - My change description (#PR)
--->
+-->
+
+- Added the option to exclude test data from the `/functions` endpoint API response (#10943)
diff --git a/src/WebJobs.Script.WebHost/Extensions/FunctionMetadataExtensions.cs b/src/WebJobs.Script.WebHost/Extensions/FunctionMetadataExtensions.cs
@@ -20,7 +20,7 @@ public static class FunctionMetadataExtensions
         /// <param name="functionMetadata">FunctionMetadata to be mapped.</param>
         /// <param name="hostOptions">The host options.</param>
         /// <returns>Promise of a FunctionMetadataResponse.</returns>
-        public static async Task<FunctionMetadataResponse> ToFunctionMetadataResponse(this FunctionMetadata functionMetadata, ScriptJobHostOptions hostOptions, string routePrefix, string baseUrl)
+        public static async Task<FunctionMetadataResponse> ToFunctionMetadataResponse(this FunctionMetadata functionMetadata, ScriptJobHostOptions hostOptions, string routePrefix, string baseUrl, bool excludeTestData)
         {
             string functionPath = GetFunctionPathOrNull(hostOptions.RootScriptPath, functionMetadata.Name);
             string functionMetadataFilePath = GetMetadataPathOrNull(functionPath);
@@ -54,7 +54,7 @@ public static async Task<FunctionMetadataResponse> ToFunctionMetadataResponse(th
                 response.ConfigHref = VirtualFileSystem.FilePathToVfsUri(functionMetadataFilePath, baseUrl, hostOptions);
             }
 
-            if (!string.IsNullOrEmpty(hostOptions.TestDataPath))
+            if (!excludeTestData && !string.IsNullOrEmpty(hostOptions.TestDataPath))
             {
                 var testDataFilePath = functionMetadata.GetTestDataFilePath(hostOptions);
                 response.TestDataHref = VirtualFileSystem.FilePathToVfsUri(testDataFilePath, baseUrl, hostOptions);
@@ -183,7 +183,7 @@ private static JObject GetFunctionConfigFromMetadata(FunctionMetadata metadata)
 
         private static async Task<string> GetTestData(string testDataPath, ScriptJobHostOptions config)
         {
-            if (!File.Exists(testDataPath))
+            if (!FileUtility.FileExists(testDataPath))
             {
                 FileUtility.EnsureDirectoryExists(Path.GetDirectoryName(testDataPath));
                 await FileUtility.WriteAsync(testDataPath, string.Empty);
diff --git a/src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs b/src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs
@@ -339,7 +339,7 @@ private async Task<SyncTriggersPayload> GetSyncTriggersPayload()
 
             // Add all listable functions details to the payload
             var listableFunctions = _functionMetadataManager.GetFunctionMetadata().Where(m => !m.IsCodeless());
-            var functionDetails = await WebFunctionsManager.GetFunctionMetadataResponse(listableFunctions, hostOptions, _hostNameProvider);
+            var functionDetails = await WebFunctionsManager.GetFunctionMetadataResponse(listableFunctions, hostOptions, _hostNameProvider, excludeTestData: _hostingConfigOptions.Value.IsTestDataSuppressionEnabled);
             result.Add("functions", new JArray(functionDetails.Select(p => JObject.FromObject(p))));
 
             // TEMP: refactor this code to properly add extensions in all scenario(#7394)
diff --git a/src/WebJobs.Script.WebHost/Management/WebFunctionsManager.cs b/src/WebJobs.Script.WebHost/Management/WebFunctionsManager.cs
@@ -8,6 +8,7 @@
 using System.Net.Http;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
+using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Management.Models;
 using Microsoft.Azure.WebJobs.Script.WebHost.Extensions;
@@ -27,8 +28,9 @@ public class WebFunctionsManager : IWebFunctionsManager
         private readonly IFunctionsSyncManager _functionsSyncManager;
         private readonly HostNameProvider _hostNameProvider;
         private readonly IFunctionMetadataManager _functionMetadataManager;
+        private readonly IOptionsMonitor<FunctionsHostingConfigOptions> _hostingConfigOptions;
 
-        public WebFunctionsManager(IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, ILoggerFactory loggerFactory, IHttpClientFactory httpClientFactory, ISecretManagerProvider secretManagerProvider, IFunctionsSyncManager functionsSyncManager, HostNameProvider hostNameProvider, IFunctionMetadataManager functionMetadataManager)
+        public WebFunctionsManager(IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, ILoggerFactory loggerFactory, IHttpClientFactory httpClientFactory, ISecretManagerProvider secretManagerProvider, IFunctionsSyncManager functionsSyncManager, HostNameProvider hostNameProvider, IFunctionMetadataManager functionMetadataManager, IOptionsMonitor<FunctionsHostingConfigOptions> hostingConfigOptions)
         {
             _applicationHostOptions = applicationHostOptions;
             _logger = loggerFactory?.CreateLogger(ScriptConstants.LogCategoryHostGeneral);
@@ -37,21 +39,22 @@ public WebFunctionsManager(IOptionsMonitor<ScriptApplicationHostOptions> applica
             _functionsSyncManager = functionsSyncManager;
             _hostNameProvider = hostNameProvider;
             _functionMetadataManager = functionMetadataManager;
+            _hostingConfigOptions = hostingConfigOptions;
         }
 
         public async Task<IEnumerable<FunctionMetadataResponse>> GetFunctionsMetadata(bool includeProxies)
         {
             var hostOptions = _applicationHostOptions.CurrentValue.ToHostOptions();
             var functionsMetadata = GetFunctionsMetadata(includeProxies, forceRefresh: false);
 
-            return await GetFunctionMetadataResponse(functionsMetadata, hostOptions, _hostNameProvider);
+            return await GetFunctionMetadataResponse(functionsMetadata, hostOptions, _hostNameProvider, excludeTestData: _hostingConfigOptions.CurrentValue.IsTestDataSuppressionEnabled);
         }
 
-        internal static async Task<IEnumerable<FunctionMetadataResponse>> GetFunctionMetadataResponse(IEnumerable<FunctionMetadata> functionsMetadata, ScriptJobHostOptions hostOptions, HostNameProvider hostNameProvider)
+        internal static async Task<IEnumerable<FunctionMetadataResponse>> GetFunctionMetadataResponse(IEnumerable<FunctionMetadata> functionsMetadata, ScriptJobHostOptions hostOptions, HostNameProvider hostNameProvider, bool excludeTestData)
         {
             string baseUrl = GetBaseUrl(hostNameProvider);
             string routePrefix = await GetRoutePrefix(hostOptions.RootScriptPath);
-            var tasks = functionsMetadata.Select(p => p.ToFunctionMetadataResponse(hostOptions, routePrefix, baseUrl));
+            var tasks = functionsMetadata.Select(p => p.ToFunctionMetadataResponse(hostOptions, routePrefix, baseUrl, excludeTestData));
 
             return await tasks.WhenAll();
         }
@@ -139,7 +142,7 @@ await functionMetadata
                 configChanged = true;
             }
 
-            if (functionMetadata.TestData != null)
+            if (functionMetadata.TestData != null && !_hostingConfigOptions.CurrentValue.IsTestDataSuppressionEnabled)
             {
                 await FileUtility.WriteAsync(dataFilePath, functionMetadata.TestData);
             }
@@ -180,7 +183,7 @@ await functionMetadata
             {
                 string routePrefix = await GetRoutePrefix(hostOptions.RootScriptPath);
                 var baseUrl = $"{request.Scheme}://{request.Host}";
-                return (true, await functionMetadata.ToFunctionMetadataResponse(hostOptions, routePrefix, baseUrl));
+                return (true, await functionMetadata.ToFunctionMetadataResponse(hostOptions, routePrefix, baseUrl, excludeTestData: _hostingConfigOptions.CurrentValue.IsTestDataSuppressionEnabled));
             }
             else
             {
diff --git a/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs b/src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs
@@ -147,6 +147,22 @@ internal bool EnableOrderedInvocationMessages
             }
         }
 
+        /// <summary>
+        /// Gets or sets a value indicating whether to ignore the TestData property during read and write operations of functions metadata.
+        /// </summary>
+        internal bool IsTestDataSuppressionEnabled
+        {
+            get
+            {
+                return GetFeatureAsBooleanOrDefault(ScriptConstants.FeatureFlagEnableTestDataSuppression, false);
+            }
+
+            set
+            {
+                _features[ScriptConstants.FeatureFlagEnableTestDataSuppression] = value ? "1" : "0";
+            }
+        }
+
         /// <summary>
         /// Gets the highest version of extension bundle v3 supported.
         /// </summary>
diff --git a/src/WebJobs.Script/ScriptConstants.cs b/src/WebJobs.Script/ScriptConstants.cs
@@ -138,6 +138,7 @@ public static class ScriptConstants
         public const string FeatureFlagEnableOrderedInvocationmessages = "EnableOrderedInvocationMessages";
         public const string FeatureFlagDisableOrderedInvocationMessages = "DisableOrderedInvocationMessages";
         public const string FeatureFlagEnableAzureMonitorTimeIsoFormat = "EnableAzureMonitorTimeIsoFormat";
+        public const string FeatureFlagEnableTestDataSuppression = "EnableTestDataSuppression";
         public const string HostingConfigDisableLinuxAppServiceDetailedExecutionEvents = "DisableLinuxExecutionDetails";
         public const string HostingConfigDisableLinuxAppServiceExecutionEventLogBackoff = "DisableLinuxLogBackoff";
         public const string FeatureFlagEnableLegacyDurableVersionCheck = "EnableLegacyDurableVersionCheck";
diff --git a/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs b/test/WebJobs.Script.Tests/Configuration/FunctionsHostingConfigOptionsTest.cs
@@ -135,7 +135,9 @@ public void Property_Validation()
                 (nameof(FunctionsHostingConfigOptions.WorkerRuntimeStrictValidationEnabled), "WORKER_RUNTIME_STRICT_VALIDATION_ENABLED=1", true),
 
                 (nameof(FunctionsHostingConfigOptions.InternalAuthApisAllowList), "InternalAuthApisAllowList=|", "|"),
-                (nameof(FunctionsHostingConfigOptions.InternalAuthApisAllowList), "InternalAuthApisAllowList=/admin/host/foo|/admin/host/bar", "/admin/host/foo|/admin/host/bar")
+                (nameof(FunctionsHostingConfigOptions.InternalAuthApisAllowList), "InternalAuthApisAllowList=/admin/host/foo|/admin/host/bar", "/admin/host/foo|/admin/host/bar"),
+
+                (nameof(FunctionsHostingConfigOptions.IsTestDataSuppressionEnabled), "EnableTestDataSuppression=1", true)
             };
 
             // use reflection to ensure that we have a test that uses every value exposed on FunctionsHostingConfigOptions
diff --git a/test/WebJobs.Script.Tests/Extensions/FunctionMetadataExtensionsTests.cs b/test/WebJobs.Script.Tests/Extensions/FunctionMetadataExtensionsTests.cs
@@ -2,9 +2,12 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System.IO;
+using System.IO.Abstractions;
+using System.Text;
 using System.Threading.Tasks;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.WebHost.Extensions;
+using Moq;
 using Newtonsoft.Json.Linq;
 using Xunit;
 
@@ -123,7 +126,7 @@ public async Task ToFunctionMetadataResponse_WithoutFiles_ReturnsExpected()
             };
 
             AddSampleBindings(functionMetadata);
-            var result = await functionMetadata.ToFunctionMetadataResponse(options, string.Empty, null);
+            var result = await functionMetadata.ToFunctionMetadataResponse(options, string.Empty, null, excludeTestData: false);
 
             Assert.Null(result.ScriptRootPathHref);
             Assert.Null(result.ConfigHref);
@@ -133,6 +136,66 @@ public async Task ToFunctionMetadataResponse_WithoutFiles_ReturnsExpected()
             Assert.Equal("httpTrigger", binding[0]["type"].Value<string>());
         }
 
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public async Task ToFunctionMetadataResponse_TestData_Exclusion(bool shouldExcludeTestDataFromApiResponse)
+        {
+            var functionName = "TestFunction1";
+            var functionMetadata = new FunctionMetadata
+            {
+                Name = functionName
+            };
+            var testDataFileName = $"{functionName}.dat";
+            var testDataFilePath = Path.Combine(_testRootScriptPath, testDataFileName);
+            var options = new ScriptJobHostOptions
+            {
+                RootScriptPath = _testRootScriptPath,
+                TestDataPath = testDataFilePath
+            };
+
+            IFileSystem fileSystem = FileUtility.Instance;
+
+            try
+            {
+                var testDataContent = @"
+                                    {
+                                        ""method"": ""POST"",
+                                        ""headers"": [],
+                                        ""body"": ""foo""
+                                    }";
+
+                var memoryStream = new MemoryStream(Encoding.UTF8.GetBytes(testDataContent))
+                {
+                    Position = 0
+                };
+                // Mock the file system
+                var mockFileSystem = new Mock<IFileSystem>();
+                var mockFile = new Mock<FileBase>();
+                mockFileSystem.Setup(f => f.Directory.Exists(It.IsAny<string>())).Returns(true);
+                mockFile.Setup(f => f.Exists(It.Is<string>(path => path.EndsWith(Path.Combine(functionName, "function.json"))))).Returns(true);
+                mockFile.Setup(f => f.Exists(It.Is<string>(path => path.EndsWith(testDataFileName)))).Returns(true);
+                mockFile.Setup(f => f.Open(It.Is<string>(path => path.EndsWith(testDataFileName)), It.IsAny<FileMode>(), It.IsAny<FileAccess>(), It.IsAny<FileShare>()))
+                        .Returns(memoryStream);
+                mockFileSystem.Setup(fs => fs.File).Returns(mockFile.Object);
+                FileUtility.Instance = mockFileSystem.Object;
+
+                AddSampleBindings(functionMetadata);
+                var result = await functionMetadata.ToFunctionMetadataResponse(options, string.Empty, null, excludeTestData: shouldExcludeTestDataFromApiResponse);
+
+                Assert.Equal(functionName, result.Name);
+                Assert.Equal(shouldExcludeTestDataFromApiResponse, result.TestData == null);
+                if (!shouldExcludeTestDataFromApiResponse)
+                {
+                    Assert.True(result.TestData.Trim().Length > 0);
+                }
+            }
+            finally
+            {
+                FileUtility.Instance = fileSystem;
+            }
+        }
+
         [Fact]
         public void GetFunctionInvokeUrlTemplate_ReturnsExpectedResult()
         {
diff --git a/test/WebJobs.Script.Tests/Managment/WebFunctionsManagerTests.cs b/test/WebJobs.Script.Tests/Managment/WebFunctionsManagerTests.cs

Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,7 @@ public static class FunctionMetadataExtensions`
`20`	`20`	`/// <param name="functionMetadata">FunctionMetadata to be mapped.</param>`
`21`	`21`	`/// <param name="hostOptions">The host options.</param>`
`22`	`22`	`/// <returns>Promise of a FunctionMetadataResponse.</returns>`
`23`		`- public static async Task<FunctionMetadataResponse> ToFunctionMetadataResponse(this FunctionMetadata functionMetadata, ScriptJobHostOptions hostOptions, string routePrefix, string baseUrl)`
	`23`	`+ public static async Task<FunctionMetadataResponse> ToFunctionMetadataResponse(this FunctionMetadata functionMetadata, ScriptJobHostOptions hostOptions, string routePrefix, string baseUrl, bool excludeTestData)`
`24`	`24`	`{`
`25`	`25`	`string functionPath = GetFunctionPathOrNull(hostOptions.RootScriptPath, functionMetadata.Name);`
`26`	`26`	`string functionMetadataFilePath = GetMetadataPathOrNull(functionPath);`
`@@ -54,7 +54,7 @@ public static async Task<FunctionMetadataResponse> ToFunctionMetadataResponse(th`
`54`	`54`	`response.ConfigHref = VirtualFileSystem.FilePathToVfsUri(functionMetadataFilePath, baseUrl, hostOptions);`
`55`	`55`	`}`
`56`	`56`
`57`		`- if (!string.IsNullOrEmpty(hostOptions.TestDataPath))`
	`57`	`+ if (!excludeTestData && !string.IsNullOrEmpty(hostOptions.TestDataPath))`
`58`	`58`	`{`
`59`	`59`	`var testDataFilePath = functionMetadata.GetTestDataFilePath(hostOptions);`
`60`	`60`	`response.TestDataHref = VirtualFileSystem.FilePathToVfsUri(testDataFilePath, baseUrl, hostOptions);`
`@@ -183,7 +183,7 @@ private static JObject GetFunctionConfigFromMetadata(FunctionMetadata metadata)`
`183`	`183`
`184`	`184`	`private static async Task<string> GetTestData(string testDataPath, ScriptJobHostOptions config)`
`185`	`185`	`{`
`186`		`- if (!File.Exists(testDataPath))`
	`186`	`+ if (!FileUtility.FileExists(testDataPath))`
`187`	`187`	`{`
`188`	`188`	`FileUtility.EnsureDirectoryExists(Path.GetDirectoryName(testDataPath));`
`189`	`189`	`await FileUtility.WriteAsync(testDataPath, string.Empty);`
Original file line number	Diff line number	Diff line change
`@@ -8,6 +8,7 @@`
`8`	`8`	`using System.Net.Http;`
`9`	`9`	`using System.Threading.Tasks;`
`10`	`10`	`using Microsoft.AspNetCore.Http;`
	`11`	`+using Microsoft.Azure.WebJobs.Script.Config;`
`11`	`12`	`using Microsoft.Azure.WebJobs.Script.Description;`
`12`	`13`	`using Microsoft.Azure.WebJobs.Script.Management.Models;`
`13`	`14`	`using Microsoft.Azure.WebJobs.Script.WebHost.Extensions;`
`@@ -27,8 +28,9 @@ public class WebFunctionsManager : IWebFunctionsManager`
`27`	`28`	`private readonly IFunctionsSyncManager _functionsSyncManager;`
`28`	`29`	`private readonly HostNameProvider _hostNameProvider;`
`29`	`30`	`private readonly IFunctionMetadataManager _functionMetadataManager;`
	`31`	`+ private readonly IOptionsMonitor<FunctionsHostingConfigOptions> _hostingConfigOptions;`
`30`	`32`
`31`		`- public WebFunctionsManager(IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, ILoggerFactory loggerFactory, IHttpClientFactory httpClientFactory, ISecretManagerProvider secretManagerProvider, IFunctionsSyncManager functionsSyncManager, HostNameProvider hostNameProvider, IFunctionMetadataManager functionMetadataManager)`
	`33`	`+ public WebFunctionsManager(IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, ILoggerFactory loggerFactory, IHttpClientFactory httpClientFactory, ISecretManagerProvider secretManagerProvider, IFunctionsSyncManager functionsSyncManager, HostNameProvider hostNameProvider, IFunctionMetadataManager functionMetadataManager, IOptionsMonitor<FunctionsHostingConfigOptions> hostingConfigOptions)`
`32`	`34`	`{`
`33`	`35`	`_applicationHostOptions = applicationHostOptions;`
`34`	`36`	`_logger = loggerFactory?.CreateLogger(ScriptConstants.LogCategoryHostGeneral);`
`@@ -37,21 +39,22 @@ public WebFunctionsManager(IOptionsMonitor<ScriptApplicationHostOptions> applica`
`37`	`39`	`_functionsSyncManager = functionsSyncManager;`
`38`	`40`	`_hostNameProvider = hostNameProvider;`
`39`	`41`	`_functionMetadataManager = functionMetadataManager;`
	`42`	`+ _hostingConfigOptions = hostingConfigOptions;`
`40`	`43`	`}`
`41`	`44`
`42`	`45`	`public async Task<IEnumerable<FunctionMetadataResponse>> GetFunctionsMetadata(bool includeProxies)`
`43`	`46`	`{`
`44`	`47`	`var hostOptions = _applicationHostOptions.CurrentValue.ToHostOptions();`
`45`	`48`	`var functionsMetadata = GetFunctionsMetadata(includeProxies, forceRefresh: false);`
`46`	`49`
`47`		`- return await GetFunctionMetadataResponse(functionsMetadata, hostOptions, _hostNameProvider);`
	`50`	`+ return await GetFunctionMetadataResponse(functionsMetadata, hostOptions, _hostNameProvider, excludeTestData: _hostingConfigOptions.CurrentValue.IsTestDataSuppressionEnabled);`
`48`	`51`	`}`
`49`	`52`
`50`		`- internal static async Task<IEnumerable<FunctionMetadataResponse>> GetFunctionMetadataResponse(IEnumerable<FunctionMetadata> functionsMetadata, ScriptJobHostOptions hostOptions, HostNameProvider hostNameProvider)`
	`53`	`+ internal static async Task<IEnumerable<FunctionMetadataResponse>> GetFunctionMetadataResponse(IEnumerable<FunctionMetadata> functionsMetadata, ScriptJobHostOptions hostOptions, HostNameProvider hostNameProvider, bool excludeTestData)`
`51`	`54`	`{`
`52`	`55`	`string baseUrl = GetBaseUrl(hostNameProvider);`
`53`	`56`	`string routePrefix = await GetRoutePrefix(hostOptions.RootScriptPath);`
`54`		`- var tasks = functionsMetadata.Select(p => p.ToFunctionMetadataResponse(hostOptions, routePrefix, baseUrl));`
	`57`	`+ var tasks = functionsMetadata.Select(p => p.ToFunctionMetadataResponse(hostOptions, routePrefix, baseUrl, excludeTestData));`
`55`	`58`
`56`	`59`	`return await tasks.WhenAll();`
`57`	`60`	`}`
`@@ -139,7 +142,7 @@ await functionMetadata`
`139`	`142`	`configChanged = true;`
`140`	`143`	`}`
`141`	`144`
`142`		`- if (functionMetadata.TestData != null)`
	`145`	`+ if (functionMetadata.TestData != null && !_hostingConfigOptions.CurrentValue.IsTestDataSuppressionEnabled)`
`143`	`146`	`{`
`144`	`147`	`await FileUtility.WriteAsync(dataFilePath, functionMetadata.TestData);`
`145`	`148`	`}`
`@@ -180,7 +183,7 @@ await functionMetadata`
`180`	`183`	`{`
`181`	`184`	`string routePrefix = await GetRoutePrefix(hostOptions.RootScriptPath);`
`182`	`185`	`var baseUrl = $"{request.Scheme}://{request.Host}";`
`183`		`- return (true, await functionMetadata.ToFunctionMetadataResponse(hostOptions, routePrefix, baseUrl));`
	`186`	`+ return (true, await functionMetadata.ToFunctionMetadataResponse(hostOptions, routePrefix, baseUrl, excludeTestData: _hostingConfigOptions.CurrentValue.IsTestDataSuppressionEnabled));`
`184`	`187`	`}`
`185`	`188`	`else`
`186`	`189`	`{`