Skip to content

Commit d5f42d0

Browse files
benbpbenbp
authored
Run smoke tests against latest published versions in release pipeline (Azure#17524)
* Run smoke tests against latest published versions in release pipeline * Update AzureAuthorityHosts reference in smoke tests * Use env variable for RunSmokeTests pass through * Only smoke test on release during manual build run * Update smoke test eventhubs* dependency to 5.3.0-beta.* * Smoke test: Use daily instead of nightly switch. Use only devops feed * [smoke tests] Remove package exclusions. Only include GA/beta packages for daily==false * Use latest non-alpha versions for smoke test csproj defaults * Use build artifact version in smoke test after release * Minor fixes for PR comments * Bump smoke test dependency * Add DoNotUpdate attribute for smoke test dependency * [smoke test] Move build artifact version override logic into Update-Dependencies script * [smoke test] Fix daily override conditional and bump core version * Fix azure authority host references * path fixes * Use ubuntu 18 vmimage for smoke test eligibility check job * Add net50 to smoke test matrix
1 parent e70cfcb commit d5f42d0

File tree

6 files changed

+270
-172
lines changed

6 files changed

+270
-172
lines changed

‎common/SmokeTests/SmokeTest/KeyVaultTest.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,10 @@ class KeyVaultTest
1414
{
1515
private static Dictionary<string, Uri> authorityHostMap = new Dictionary<string, Uri>
1616
{
17-
{ "AzureCloud", KnownAuthorityHosts.AzureCloud },
18-
{ "AzureChinaCloud", KnownAuthorityHosts.AzureChinaCloud },
19-
{ "AzureGermanCloud", KnownAuthorityHosts.AzureGermanCloud },
20-
{ "AzureUSGovernment", KnownAuthorityHosts.AzureUSGovernment },
17+
{ "AzureCloud", AzureAuthorityHosts.AzurePublicCloud },
18+
{ "AzureChinaCloud", AzureAuthorityHosts.AzureChina },
19+
{ "AzureGermanCloud", AzureAuthorityHosts.AzureGermany },
20+
{ "AzureUSGovernment", AzureAuthorityHosts.AzureGovernment },
2121
};
2222

2323
private static string SecretName = $"SmokeTestSecret-{Guid.NewGuid()}";
@@ -38,7 +38,7 @@ public static async Task RunTests()
3838
Console.WriteLine("3.- Delete that Secret (Clean up)\n");
3939

4040
string keyVaultUri = Environment.GetEnvironmentVariable("KEY_VAULT_URI");
41-
var authorityHost = GetAuthorityHost(Environment.GetEnvironmentVariable("AZURE_CLOUD"), KnownAuthorityHosts.AzureCloud);
41+
var authorityHost = GetAuthorityHost(Environment.GetEnvironmentVariable("AZURE_CLOUD"), AzureAuthorityHosts.AzurePublicCloud);
4242

4343
var defaultAzureCredentialOptions = new DefaultAzureCredentialOptions
4444
{
@@ -95,4 +95,4 @@ private static Uri GetAuthorityHost(string cloudName, Uri defaultAuthorityHost)
9595
return defaultAuthorityHost;
9696
}
9797
}
98-
}
98+
}

‎common/SmokeTests/SmokeTest/SmokeTest.csproj‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,26 +5,24 @@
55
<LangVersion>latest</LangVersion>
66
<StartupObject>SmokeTest.Program</StartupObject>
77
</PropertyGroup>
8-
98
<ItemGroup Condition="'$(TargetFramework)' == 'net461'">
109
<Reference Include="System" />
1110
<Reference Include="Microsoft.CSharp" />
1211
</ItemGroup>
13-
1412
<ItemGroup>
15-
<PackageReference Include="Azure.Core" Version="1.4.0-dev.*" />
16-
<PackageReference Include="Azure.Identity" Version="1.2.0-dev.*" />
17-
<PackageReference Include="Azure.Messaging.EventHubs" Version="5.1.0-dev.*" />
18-
<PackageReference Include="Azure.Messaging.EventHubs.Processor" Version="5.1.0-dev.*" />
19-
<PackageReference Include="Azure.Security.Keyvault.Secrets" Version="4.1.0-dev.*" />
20-
<PackageReference Include="Azure.Storage.Blobs" Version="12.5.0-dev.*" />
21-
<PackageReference Include="Microsoft.Azure.Amqp" Version="2.4.3" />
13+
<PackageReference Include="Azure.Core" Version="1.8.0" />
14+
<PackageReference Include="Azure.Identity" Version="1.4.0-beta.1" />
15+
<!-- The OverrideDailyVersion attribute prevents the Update-Dependencies script from overwriting it with a daily build version -->
16+
<PackageReference Include="Azure.Messaging.EventHubs" Version="5.3.0-beta.4" OverrideDailyVersion="" />
17+
<PackageReference Include="Azure.Messaging.EventHubs.Processor" Version="5.3.0-beta.4" />
18+
<PackageReference Include="Azure.Security.Keyvault.Secrets" Version="4.2.0-beta.3" />
19+
<PackageReference Include="Azure.Storage.Blobs" Version="12.8.0-beta.1" />
20+
<PackageReference Include="Microsoft.Azure.Amqp" Version="2.4.9" />
2221
<PackageReference Include="Microsoft.Azure.Devices" Version="1.20.1" />
2322
<PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.10.0" />
2423
</ItemGroup>
25-
2624
<!-- Sample: IoT Hub Connection String Translation -->
2725
<ItemGroup>
2826
<Compile Include="$(MSBuildThisFileDirectory)..\..\..\samples\iothub-connect-to-eventhubs\**\*.cs" Link="Samples\IoTHubConnection\SharedSource\%(Filename)%(Extension)" />
2927
</ItemGroup>
30-
</Project>
28+
</Project>

‎common/SmokeTests/SmokeTest/Update-Dependencies.ps1‎

Lines changed: 87 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,21 @@
11
param(
22
[string]$ProjectFile = './SmokeTest.csproj',
3+
[string]$ArtifactsPath,
34
[switch]$SkipVersionValidation,
4-
[switch]$CI
5+
[switch]$CI,
6+
[switch]$Daily
57
)
68

7-
# To exclude a package version create an entry whose key is the package to
8-
# exclude whose value is a hash table of versions to exclude.
9-
# Example:
10-
# $PACKAGE_EXCLUSIONS = @{
11-
# 'Azure.Security.Keyvault.Secrets' = @{
12-
# '4.1.0-dev.20191102.1' = $true;
13-
# '4.1.0-dev.20191103.1' = $true;
14-
# }
15-
# }
16-
$PACKAGE_EXCLUSIONS = @{ }
9+
. $PSScriptRoot/../../../eng/common/scripts/SemVer.ps1
1710

1811
$PACKAGE_REFERENCE_XPATH = '//Project/ItemGroup/PackageReference'
1912

2013
# Matches the dev.yyyymmdd portion of the version string
21-
$DEV_DATE_REGEX = 'dev\.(\d{8})'
14+
$ALPHA_DATE_REGEX = 'alpha\.(\d{8})'
2215

23-
$NIGHTLY_FEED_NAME = 'NightlyFeed'
24-
$NIGHTLY_FEED_URL = 'https://azuresdkartifacts.blob.core.windows.net/azure-sdk-for-net/index.json'
16+
$baselineVersionDate = $null;
17+
18+
$PACKAGE_FEED_URL = 'https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json'
2519

2620
function Log-Warning($message) {
2721
if ($CI) {
@@ -31,63 +25,94 @@ function Log-Warning($message) {
3125
}
3226
}
3327

34-
Register-PackageSource `
35-
-Name $NIGHTLY_FEED_NAME `
36-
-Location $NIGHTLY_FEED_URL `
37-
-ProviderName Nuget `
38-
-ErrorAction SilentlyContinue `
28+
function GetAllPackages {
29+
$packages = Find-Package -Source $PACKAGE_FEED_URL -AllVersion -AllowPrereleaseVersions
30+
if ($Daily) {
31+
return $packages | Where-Object { $_.Version.Contains("alpha") }
32+
}
33+
return $packages | Where-Object { !$_.Version.Contains("alpha") -and !$_.Version.Contains("dev") }
34+
}
3935

40-
# List all packages from the source specified by $FeedName. Packages are sorted
41-
# ascending by version according to semver rules (e.g. 4.0.0-preview.1 comes
42-
# before 4.0.0) not lexicographically.
43-
# Packages cannot be filtered at this stage because the sleet feed to which they
44-
# are published does not support filtering by name.
45-
$allPackages = Find-Package -Source $NIGHTLY_FEED_NAME -AllVersion -AllowPrereleaseVersions
36+
function GetLatestPackage([array]$packageList, [string]$packageName) {
37+
$versions = ($packageList
38+
| Where-Object { $_.Name -eq $packageName }
39+
| Select-Object -ExpandProperty Version)
4640

47-
$baselineVersionDate = $null;
41+
if (!$versions) {
42+
Write-Warning "Did not find any versions for $($packageName)"
43+
return
44+
}
4845

49-
# For each PackageReferecne in the csproj, find the latest version of that
50-
# package from the dev feed which is not in the excluded list.
51-
$projectFilePath = Resolve-Path -Path $ProjectFile
52-
[xml]$csproj = Get-Content $ProjectFile
53-
$csproj |
54-
Select-XML $PACKAGE_REFERENCE_XPATH |
55-
Where-Object { $_.Node.HasAttribute('Version') } |
56-
ForEach-Object {
57-
# Resolve package version:
58-
$packageName = $_.Node.Include
59-
60-
# This assumes that the versions coming back from Find-Package are
61-
# sorted ascending. It excludes any version that is in the corresponding
62-
# $PACKAGE_EXCLUSIONS entry
63-
$targetVersion = ($allPackages |
64-
Where-Object { $_.Name -eq $packageName } |
65-
Where-Object { -not ( $PACKAGE_EXCLUSIONS.ContainsKey($packageName) -and $PACKAGE_EXCLUSIONS[$packageName].ContainsKey($_.Version)) } |
66-
Select-Object -Last 1).Version
67-
68-
if ($targetVersion -eq $null) {
69-
return
70-
}
46+
$sorted = [AzureEngSemanticVersion]::SortVersionStrings($versions)
47+
return $sorted | Select-Object -First 1
48+
}
7149

72-
Write-Host "Setting $packageName to $targetVersion"
73-
$_.Node.Version = "$targetVersion"
50+
function GetPackageVersion([array]$packageList, [string]$packageName) {
51+
if ($Daily -or -not $ArtifactsPath) {
52+
return GetLatestPackage $packageList $packageName
53+
}
7454

55+
if (-not (Test-Path (Join-Path $ArtifactsPath $packageName))) {
56+
Write-Host "No build artifact directory for smoke test dependency $packageName. Using latest upstream version."
57+
return GetLatestPackage $packageList $packageName
58+
}
7559

76-
# Validate package version date component matches
77-
if ($SkipVersionValidation) {
78-
return
79-
}
60+
$pkg = Get-ChildItem "$ArtifactsPath/$packageName/*.nupkg" | Select-Object -First 1
61+
if ($pkg -match "$packageName\.(.*)\.nupkg") {
62+
$version = $matches[1]
63+
Write-Host "Found build artifact for $packageName with version $version. Using artifact version."
64+
return $version
65+
} else {
66+
throw "No build artifact packages found for smoke test dependency '$packageName'."
67+
}
68+
}
8069

81-
if ($_.Node.Version -match $DEV_DATE_REGEX) {
82-
if ($baselineVersionDate -eq $null) {
83-
Write-Host "Using baseline version date: $($matches[1])"
84-
$baselineVersionDate = $matches[1]
70+
function SetLatestPackageVersions([xml]$csproj) {
71+
# For each PackageReference in the csproj, find the latest version of that
72+
# package from the dev feed which is not in the excluded list.
73+
$allPackages = GetAllPackages
74+
$csproj |
75+
Select-XML $PACKAGE_REFERENCE_XPATH
76+
| Where-Object { $_.Node.HasAttribute('Version') }
77+
| Where-Object { -not ($Daily -and $_.Node.HasAttribute('OverrideDailyVersion')) }
78+
| ForEach-Object {
79+
# Resolve package version:
80+
$packageName = $_.Node.Include
81+
82+
$targetVersion = GetPackageVersion $allPackages $packageName
83+
84+
if ($null -eq $targetVersion) {
85+
return
8586
}
8687

87-
if ($baselineVersionDate -ne $matches[1]) {
88-
Log-Warning "$($_.Node.Include) uses invalid version. Expected: $baselineVersionDate, Actual: $($matches[1])"
88+
Write-Host "Setting $packageName to $targetVersion"
89+
$_.Node.Version = "$targetVersion"
90+
91+
92+
# Validate package version date component matches
93+
if ($SkipVersionValidation) {
94+
return
95+
}
96+
97+
if ($_.Node.Version -match $ALPHA_DATE_REGEX) {
98+
$capture = $matches[1]
99+
if ($null -eq $baselineVersionDate) {
100+
Write-Host "Using baseline version date: $capture"
101+
$baselineVersionDate = $capture
102+
}
103+
104+
if ($baselineVersionDate -ne $matches[1]) {
105+
Log-Warning "$($_.Node.Include) uses invalid version. Expected: $baselineVersionDate, Actual: $capture"
106+
}
89107
}
90108
}
91-
}
109+
}
110+
111+
function UpdateCsprojVersions {
112+
$projectFilePath = Resolve-Path -Path $ProjectFile
113+
[xml]$csproj = Get-Content $projectFilePath
114+
SetLatestPackageVersions $csproj
115+
$csproj.Save($projectFilePath)
116+
}
92117

93-
$csproj.Save($projectFilePath)
118+
UpdateCsprojVersions

‎common/SmokeTests/smoke-tests.yml‎

Lines changed: 3 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -2,95 +2,6 @@ variables:
22
- template: /eng/pipelines/templates/variables/globals.yml
33

44
jobs:
5-
- job: SmokeTest
6-
7-
strategy:
8-
matrix:
9-
Linux (AzureCloud):
10-
OSVmImage: "ubuntu-18.04"
11-
TestTargetFramework: netcoreapp2.1
12-
SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
13-
ArmTemplateParameters: $(azureCloudArmParameters)
14-
Linux (AzureCloud Canary):
15-
OSVmImage: "ubuntu-18.04"
16-
TestTargetFramework: netcoreapp2.1
17-
SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources-preview)
18-
ArmTemplateParameters: $(azureCloudArmParameters)
19-
Location: "eastus2euap"
20-
Windows_NetCoreApp (AzureCloud):
21-
OSVmImage: "windows-2019"
22-
TestTargetFramework: netcoreapp2.1
23-
SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
24-
ArmTemplateParameters: $(azureCloudArmParameters)
25-
Windows_NetFramework (AzureCloud):
26-
OSVmImage: "windows-2019"
27-
TestTargetFramework: net461
28-
SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
29-
ArmTemplateParameters: $(azureCloudArmParameters)
30-
MacOs (AzureCloud):
31-
OSVmImage: "macOS-10.15"
32-
TestTargetFramework: netcoreapp2.1
33-
SubscriptionConfiguration: $(sub-config-azure-cloud-test-resources)
34-
ArmTemplateParameters: $(azureCloudArmParameters)
35-
Windows_NetCoreApp (AzureUSGovernment):
36-
OSVmImage: "windows-2019"
37-
TestTargetFramework: netcoreapp2.1
38-
SubscriptionConfiguration: $(sub-config-gov-test-resources)
39-
ArmTemplateParameters: $(azureUSGovernmentArmParameters)
40-
Windows_NetFramework (AzureUSGovernment):
41-
OSVmImage: "windows-2019"
42-
TestTargetFramework: net461
43-
SubscriptionConfiguration: $(sub-config-gov-test-resources)
44-
ArmTemplateParameters: $(azureUSGovernmentArmParameters)
45-
Windows_NetCoreApp (AzureChinaCloud):
46-
OSVmImage: "windows-2019"
47-
TestTargetFramework: netcoreapp2.1
48-
SubscriptionConfiguration: $(sub-config-cn-test-resources)
49-
ArmTemplateParameters: $(azureChinaCloudArmParameters)
50-
Windows_NetFramework (AzureChinaCloud):
51-
OSVmImage: "windows-2019"
52-
TestTargetFramework: net461
53-
SubscriptionConfiguration: $(sub-config-cn-test-resources)
54-
ArmTemplateParameters: $(azureChinaCloudArmParameters)
55-
56-
pool:
57-
vmImage: $(OSVmImage)
58-
59-
variables:
60-
azureCloudArmParameters: "@{ storageEndpointSuffix = 'core.windows.net'; azureCloud = 'AzureCloud'; }"
61-
azureUSGovernmentArmParameters: "@{ storageEndpointSuffix = 'core.usgovcloudapi.net'; azureCloud = 'AzureUSGovernment'; }"
62-
azureChinaCloudArmParameters: "@{ storageEndpointSuffix = 'core.chinacloudapi.cn'; azureCloud = 'AzureChinaCloud'; }"
63-
64-
steps:
65-
- template: /eng/common/pipelines/templates/steps/verify-agent-os.yml
66-
67-
- task: PowerShell@2
68-
inputs:
69-
targetType: filePath
70-
filePath: ./common/SmokeTests/SmokeTest/Update-Dependencies.ps1
71-
arguments: -CI
72-
workingDirectory: common/SmokeTests/SmokeTest
73-
pwsh: true
74-
displayName: Use latest dev feed package versions
75-
76-
- pwsh: Get-Content ./common/SmokeTests/SmokeTest/SmokeTest.csproj
77-
displayName: Show SmokeTest.csproj
78-
79-
- template: /eng/pipelines/templates/steps/install-dotnet.yml
80-
81-
- pwsh: dotnet restore ./common/SmokeTests/SmokeTest/SmokeTest.csproj
82-
displayName: dotnet restore
83-
84-
- template: /eng/common/TestResources/deploy-test-resources.yml
85-
parameters:
86-
ServiceDirectory: '$(Build.SourcesDirectory)/common/SmokeTests/'
87-
SubscriptionConfiguration: $(SubscriptionConfiguration)
88-
ArmTemplateParameters: $(ArmTemplateParameters)
89-
90-
- pwsh: dotnet run -p .\common\SmokeTests\SmokeTest\SmokeTest.csproj --framework $(TestTargetFramework)
91-
displayName: "Run Smoke Tests"
92-
93-
- template: /eng/common/TestResources/remove-test-resources.yml
94-
parameters:
95-
ServiceDirectory: '$(Build.SourcesDirectory)/common/SmokeTests/'
96-
SubscriptionConfiguration: $(SubscriptionConfiguration)
5+
- template: /eng/pipelines/templates/jobs/smoke-tests.yml
6+
parameters:
7+
Daily: true

0 commit comments

Comments
 (0)