diff --git a/DECISIONS.md b/DECISIONS.md new file mode 100644 index 0000000000..3c8fca1001 --- /dev/null +++ b/DECISIONS.md @@ -0,0 +1,115 @@ +# Design Decisions + +This document records significant design decisions made during the implementation of the F# Compiler Regression Testing pipeline. + +## Decision 1: Template-Based Architecture + +**Context**: Need to implement regression testing that can be reused across pipelines. + +**Options Considered**: +1. Inline job definitions in azure-pipelines-PR.yml +2. Reusable Azure DevOps template in eng/templates/ + +**Decision**: Use reusable template approach. + +**Rationale**: +- Follows Azure DevOps best practices +- Reduces code duplication +- Makes it easy to extend with new libraries +- Consistent with existing patterns in the repository (eng/common/templates/) + +--- + +## Decision 2: Optimized Artifact Publishing + +**Context**: Need to share compiler artifacts between jobs. + +**Options Considered**: +1. Publish entire artifacts folder (~1.8GB) +2. Publish only essential directories (fsc and FSharp.Core) (~79MB) + +**Decision**: Publish only essential directories. + +**Rationale**: +- Reduces artifact size from 1.8GB to ~79MB +- Faster artifact upload/download +- Contains all necessary components for regression testing +- Matches approach in previous PR #18803 + +--- + +## Decision 3: Using F# Script for Directory.Build.props Setup + +**Context**: Need to inject UseLocalCompiler.Directory.Build.props import into third-party repos. + +**Options Considered**: +1. Pure PowerShell XML manipulation +2. F# script with proper XML handling +3. Simple file replacement + +**Decision**: Use F# script with XML handling. + +**Rationale**: +- Properly handles existing Directory.Build.props files +- Correctly inserts import at beginning of Project element +- Native F# tooling in an F# project +- Matches approach in previous PR #18803 + +--- + +## Decision 4: Specific Commit SHAs for Third-Party Libraries + +**Context**: Need reproducible regression tests. + +**Options Considered**: +1. Use main/master branch +2. Use specific commit SHAs + +**Decision**: Use specific commit SHAs. + +**Rationale**: +- Ensures reproducible test results +- Protects against breaking changes in third-party libraries +- Allows controlled updates when ready +- Standard practice for regression testing + +--- + +## Decision 5: Removal of Strategy Matrix from EndToEndBuildTests + +**Context**: The original EndToEndBuildTests job had a matrix for regular vs experimental features. + +**Options Considered**: +1. Keep the matrix and publish artifacts only from one configuration +2. Remove the matrix entirely +3. Publish artifacts from both configurations + +**Decision**: Remove the matrix entirely (per previous PR approach). + +**Rationale**: +- Simplifies artifact publishing +- Regression tests need consistent baseline +- Both configurations were building with empty experimental flag anyway +- Matches approach in previous PR #18803 + +--- + +## Decision 6: Use net10.0 Target Framework in Template + +**Context**: The pipeline needs to reference the correct .NET target framework for the compiler artifacts. + +**Options Considered**: +1. Hardcode net9.0 (as in PR #18803) +2. Use net10.0 (per current UseLocalCompiler.Directory.Build.props) +3. Make it configurable + +**Decision**: Use net10.0 to match the current project state. + +**Rationale**: +- The current repository uses .NET 10 SDK +- UseLocalCompiler.Directory.Build.props references net10.0 paths +- The artifacts/bin/fsc/Release/net10.0 directory exists +- PR #18803 used net9.0 because it was based on an older version of the repository +- Using net10.0 ensures compatibility with the current build output + +--- diff --git a/OBSTACLES.md b/OBSTACLES.md new file mode 100644 index 0000000000..502da94973 --- /dev/null +++ b/OBSTACLES.md @@ -0,0 +1,79 @@ +# Obstacles Encountered + +This document tracks any obstacles encountered during the implementation of the Azure DevOps regression testing template. + +## Current Obstacles + +None at this time. + +## Resolved Obstacles + +### 1. FSharpPlus Build Failure: `_GetRestoreSettingsPerFramework` Target Missing + +**Date:** 2025-11-28 + +**Symptom:** +``` +D:\a\_work\1\TestRepo\src\FSharpPlus.TypeLevel\Providers\FSharpPlus.Providers.fsproj : error MSB4057: The target "_GetRestoreSettingsPerFramework" does not exist in the project. [TargetFramework=net8.0] +``` + +**Root Cause:** +The artifacts were being downloaded to an incorrect path structure. The UseLocalCompiler.Directory.Build.props file expects: +```xml +$(LocalFSharpCompilerPath)/artifacts/bin/fsc/$(LocalFSharpCompilerConfiguration)/net10.0 +``` + +But the artifacts were being downloaded to: +``` +$(Pipeline.Workspace)/FSharpCompiler/bin/fsc (missing /artifacts/) +``` + +This caused the FSharpTargetsShim property to point to a non-existent path, which prevented Microsoft.FSharp.Targets from being imported. Without Microsoft.FSharp.Targets, Microsoft.Common.targets was not imported, which in turn prevented NuGet.targets from being imported - and NuGet.targets is where `_GetRestoreSettingsPerFramework` is defined. + +**Investigation Steps:** +1. Used `dotnet msbuild -pp` to compare preprocessed output with and without local compiler +2. Found NuGet.targets was referenced 9 times without local compiler, but only 2 times with local compiler +3. Traced the import chain: FSharpTargetsShim -> Microsoft.FSharp.NetSdk.targets -> Microsoft.FSharp.Targets -> Microsoft.Common.targets -> NuGet.targets +4. Discovered FSharpTargetsShim was pointing to path with doubled `/artifacts/artifacts/` +5. Realized artifact download path didn't match UseLocalCompiler.Directory.Build.props expectations + +**Solution:** +Changed the artifact download paths to include `/artifacts/bin/`: +```yaml +downloadPath: '$(Pipeline.Workspace)/FSharpCompiler/artifacts/bin/fsc' +downloadPath: '$(Pipeline.Workspace)/FSharpCompiler/artifacts/bin/FSharp.Core' +``` + +This ensures the directory structure matches what UseLocalCompiler.Directory.Build.props expects. + +**Lessons Learned:** +- UseLocalCompiler.Directory.Build.props has specific path expectations that must be matched exactly +- Use `dotnet msbuild -pp` to debug MSBuild import issues +- The `_GetRestoreSettingsPerFramework` error often indicates broken MSBuild import chain + +### 2. Git Submodule Initialization + +**Date:** 2025-11-28 + +**Symptom:** +Type provider source files were missing. + +**Root Cause:** +FSharpPlus uses git submodules for `FSharp.TypeProviders.SDK`. + +**Solution:** +1. Changed `git clone` to `git clone --recursive` to clone with submodules +2. Added explicit `git submodule update --init --recursive` after `git checkout` + +### 3. No Binary Log Files Generated + +**Date:** 2025-11-28 + +**Symptom:** +No `.binlog` files were being collected or published. + +**Root Cause:** +The build was failing early before any MSBuild commands could generate binlog files. + +**Solution:** +Fixing the path issues above allows the build to proceed, enabling binlog generation. diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000000..adc652c918 --- /dev/null +++ b/TODO.md @@ -0,0 +1,68 @@ +# TODO: Azure DevOps Template for F# Compiler Regression Testing + +## Overview +Implement a reusable Azure DevOps template for F# compiler regression testing, integrated with the existing PR pipeline infrastructure. + +## Current Issues (from review feedback) + +### Build Failure & Binlog Issues +- [x] Fix binlog collection (was failing due to missing submodules) +- [x] Fix the `_GetRestoreSettingsPerFramework` issue (caused by missing git submodules) +- [x] Add `--recursive` flag to git clone command +- [x] Add explicit `git submodule update --init --recursive` after checkout +- [x] Document findings in OBSTACLES.md + +## Tasks + +### Infrastructure Setup +- [x] Analyze existing PR pipeline structure (`azure-pipelines-PR.yml`) +- [x] Review previous PR #18803 implementation details +- [x] Understand `UseLocalCompiler.Directory.Build.props` configuration + +### Implementation +- [x] Create `eng/templates/` directory +- [x] Create `eng/templates/regression-test-jobs.yml` template + - [x] Define parameters for testMatrix + - [x] Implement job that depends on EndToEndBuildTests + - [x] Add artifact download steps (FSharpCompilerFscArtifacts, FSharpCoreArtifacts, UseLocalCompilerProps) + - [x] Add third-party repo checkout step + - [x] Add .NET SDK installation step + - [x] Add Directory.Build.props setup step referencing standalone F# script + - [x] Add environment reporting step + - [x] Add build execution step + - [x] Add artifact publishing step + - [x] Add result reporting step + - [x] Add optional imageOverride per tested repo + +### F# Script for Repository Setup +- [x] Create standalone `eng/scripts/PrepareRepoForRegressionTesting.fsx` +- [x] Test script locally with FSharpPlus repository +- [x] Handle both existing and missing Directory.Build.props cases + +### Integration +- [x] Update `azure-pipelines-PR.yml`: + - [x] Modify EndToEndBuildTests to publish focused artifacts + - [x] Remove strategy/matrix section from EndToEndBuildTests + - [x] Add artifact publishing tasks for fsc, FSharp.Core, and UseLocalCompiler props + - [x] Move template invocation to stage level (outside common template) + - [x] Add template invocation with FSharpPlus test matrix + +### Documentation +- [x] Create `docs/regression-testing-pipeline.md` + - [x] Purpose and overview + - [x] How it works + - [x] Current test matrix + - [x] Adding new libraries + - [x] Pipeline configuration + - [x] Troubleshooting + - [x] Technical details + +### Validation +- [x] Verify YAML syntax is valid +- [x] Verify template structure matches Azure DevOps best practices +- [x] Ensure Release configuration is used throughout +- [x] Test F# script locally with FSharpPlus + +## References +- Previous PR: https://github.com/dotnet/fsharp/pull/18803 +- Files: `eng/templates/regression-test-jobs.yml`, `azure-pipelines-PR.yml`, `docs/regression-testing-pipeline.md`, `eng/scripts/PrepareRepoForRegressionTesting.fsx` diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index 0f8bfcea2f..cd84d88b80 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -700,35 +700,39 @@ stages: pool: name: $(DncEngPublicBuildPool) demands: ImageOverride -equals $(_WindowsMachineQueueName) - strategy: - maxParallel: 2 - matrix: - regular: - _experimental_flag: '' - experimental_features: - _experimental_flag: '' steps: - checkout: self clean: true - # We first download a publicly available .NET SDK. That one has support for `path` in global.json. dotnet.cmd script can then download a version which is not yet shipped, but matches global.json. - - task: UseDotNet@2 - displayName: install SDK - inputs: - packageType: sdk - version: '10.x' - includePreviewVersions: true - workingDirectory: $(Build.SourcesDirectory) - installationPath: $(Build.SourcesDirectory)/.dotnet - script: .\Build.cmd -c Release -pack env: NativeToolsOnMachine: true - FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c Release - env: - FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) displayName: End to end build tests - - script: .\eng\common\dotnet.cmd fsi .\tests\FSharp.Compiler.ComponentTests\CompilerCompatibilityTests.fsx - displayName: Compiler compatibility tests + + # Publish artifacts for regression testing + - task: PublishPipelineArtifact@1 + displayName: Publish F# Compiler FSC Artifacts for Regression Tests + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts/bin/fsc' + artifactName: 'FSharpCompilerFscArtifacts' + publishLocation: pipeline + condition: succeeded() + + - task: PublishPipelineArtifact@1 + displayName: Publish F# Core Artifacts for Regression Tests + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts/bin/FSharp.Core' + artifactName: 'FSharpCoreArtifacts' + publishLocation: pipeline + condition: succeeded() + + - task: PublishPipelineArtifact@1 + displayName: Publish UseLocalCompiler props file for Regression Tests + inputs: + targetPath: '$(Build.SourcesDirectory)/UseLocalCompiler.Directory.Build.props' + artifactName: 'UseLocalCompilerProps' + publishLocation: pipeline + condition: succeeded() # Up-to-date - disabled due to it being flaky #- job: UpToDate_Windows @@ -881,3 +885,20 @@ stages: - pwsh: .\tests\ILVerify\ilverify.ps1 displayName: Run ILVerify workingDirectory: $(Build.SourcesDirectory) + + #-------------------------------------------------------------------------------------------------------------------# + # F# Compiler Regression Tests # + #-------------------------------------------------------------------------------------------------------------------# + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - template: /eng/templates/regression-test-jobs.yml + parameters: + testMatrix: + - repo: fsprojects/FSharpPlus + commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 + buildScript: build.cmd + displayName: FSharpPlus_Windows + - repo: fsprojects/FSharpPlus + commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 + buildScript: build.sh + displayName: FSharpPlus_Linux + useVmImage: $(UbuntuMachineQueueName) diff --git a/docs/regression-testing-pipeline.md b/docs/regression-testing-pipeline.md new file mode 100644 index 0000000000..5e3013bfad --- /dev/null +++ b/docs/regression-testing-pipeline.md @@ -0,0 +1,174 @@ +# F# Compiler Regression Testing + +This document describes the F# compiler regression testing functionality implemented as a reusable Azure DevOps template in `eng/templates/regression-test-jobs.yml` and integrated into the main PR pipeline (`azure-pipelines-PR.yml`). + +## Purpose + +The regression testing helps catch F# compiler regressions by building popular third-party F# libraries with the freshly built compiler from this repository. This provides early detection of breaking changes that might affect real-world F# projects. + +## How It Works + +### Integration with PR Pipeline + +The regression tests are automatically run as part of every PR build, depending on the `EndToEndBuildTests` job for the F# compiler artifacts. + +### Template-Based Architecture + +The regression testing logic is implemented as a reusable Azure DevOps template that can be consumed by multiple pipelines: + +- **Template Location**: `eng/templates/regression-test-jobs.yml` +- **Integration**: Called from `azure-pipelines-PR.yml` +- **Dependencies**: Depends on `EndToEndBuildTests` job for compiler artifacts + +### Workflow + +1. **Build F# Compiler**: The `EndToEndBuildTests` job builds the F# compiler and publishes required artifacts +2. **Matrix Execution**: For each library in the test matrix (running in parallel): + - Checkout the third-party repository at a specific commit + - Install appropriate .NET SDK version using the repository's `global.json` + - Setup `Directory.Build.props` to import `UseLocalCompiler.Directory.Build.props` + - Build the library using its standard build script + - Publish MSBuild binary logs for analysis +3. **Report Results**: Success/failure status is reported with build logs for diagnosis + +### Key Features + +- **Reproducible Testing**: Uses specific commit SHAs for third-party libraries to ensure consistent results +- **Matrix Configuration**: Supports testing multiple libraries with different build requirements +- **Detailed Logging**: Captures comprehensive build logs, binary logs, and environment information +- **Artifact Publishing**: Publishes build outputs for analysis when builds fail + +## Current Test Matrix + +The pipeline currently tests against: + +| Library | Repository | Commit | Build Script | Purpose | +|---------|------------|--------|--------------|---------| +| FSharpPlus | fsprojects/FSharpPlus | f614035b75922aba41ed6a36c2fc986a2171d2b8 | build.cmd | Tests advanced F# language features | + +## Adding New Libraries + +To add a new library to the test matrix, update the template invocation in `azure-pipelines-PR.yml`: + +```yaml +# F# Compiler Regression Tests using third-party libraries +- template: /eng/templates/regression-test-jobs.yml + parameters: + testMatrix: + - repo: fsprojects/FSharpPlus + commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 + buildScript: build.cmd + displayName: FSharpPlus + - repo: your-org/your-library # Add your library here + commit: abc123def456... # Specific commit SHA + buildScript: build.sh # Build script (build.cmd, build.sh, etc.) + displayName: YourLibrary # Human-readable name +``` + +Each test matrix entry requires: +- **repo**: GitHub repository in `owner/name` format +- **commit**: Specific commit SHA for reproducible results +- **buildScript**: Build script to execute (e.g., `build.cmd`, `build.sh`) +- **displayName**: Human-readable name for the job + +## Pipeline Configuration + +### Triggers + +Regression tests run automatically as part of PR builds when: +- **PR Pipeline**: Triggered by pull requests to main branches +- **Dependencies**: Runs after `EndToEndBuildTests` completes successfully +- **Parallel Execution**: Each repository in the test matrix runs as a separate job in parallel + +### Build Environment + +- **OS**: Windows (using `$(WindowsMachineQueueName)`) +- **Pool**: Standard public build pool (`$(DncEngPublicBuildPool)`) +- **Timeout**: 60 minutes per regression test job +- **.NET SDK**: Automatically detects and installs SDK version from each repository's `global.json` + +### Artifacts + +The regression tests publish focused artifacts for analysis: +- **FSharpCompilerArtifacts**: F# compiler build output (from `EndToEndBuildTests`) +- **UseLocalCompilerProps**: Configuration file for using local compiler (from `EndToEndBuildTests`) +- **{LibraryName}_BinaryLogs**: MSBuild binary logs from each tested library for efficient diagnosis + +## Troubleshooting Build Failures + +When a regression test fails: + +1. **Check the Job Summary**: Look at the final status report for high-level information. + +2. **Download Build Logs**: Download the published artifacts to examine detailed build output. + +3. **Compare Compiler Changes**: Review what changes were made to the compiler that might affect the failing library. + +4. **Local Reproduction**: Use the `UseLocalCompiler.Directory.Build.props` file to reproduce the issue locally. + +### Local Testing + +To test a library locally with your F# compiler build: + +1. Build the F# compiler: `.\Build.cmd -c Release -pack` + +2. In the third-party library directory, create a `Directory.Build.props`: + ```xml + + + + ``` + +3. Update the `LocalFSharpCompilerPath` in `UseLocalCompiler.Directory.Build.props` to point to your F# repository. + +4. Set environment variables: + ```cmd + set LoadLocalFSharpBuild=true + set LocalFSharpCompilerConfiguration=Release + ``` + +5. Run the library's build script. + +## Best Practices + +### For Library Selection + +- **Coverage**: Choose libraries that exercise different F# language features +- **Popularity**: Include widely-used libraries that represent real-world usage +- **Stability**: Use libraries with stable build processes and minimal external dependencies +- **Diversity**: Include libraries with different build systems and target frameworks + +### For Maintenance + +- **Regular Updates**: Periodically update commit SHAs to newer stable versions +- **Monitor Dependencies**: Watch for changes in third-party library build requirements +- **Baseline Management**: Update baselines when intentional breaking changes are made + +## Technical Details + +### UseLocalCompiler.Directory.Build.props + +This MSBuild props file configures projects to use the locally built F# compiler instead of the SDK version. Key settings: + +- `LocalFSharpCompilerPath`: Points to the F# compiler artifacts +- `DotnetFscCompilerPath`: Path to the fsc.dll compiler +- `DisableImplicitFSharpCoreReference`: Ensures local FSharp.Core is used + +### Path Handling + +The pipeline dynamically updates paths in the props file using PowerShell: +```powershell +$content -replace 'LocalFSharpCompilerPath.*MSBuildThisFileDirectory.*', 'LocalFSharpCompilerPath>$(Pipeline.Workspace)/FSharpCompiler<' +``` + +This ensures the correct path is used in the Azure DevOps environment. + +## Future Enhancements + +Potential improvements to the pipeline: + +1. **Performance Testing**: Measure compilation times and memory usage +2. **Multiple Target Frameworks**: Test libraries across different .NET versions +3. **Parallel Execution**: Run library tests in parallel for faster feedback +4. **Automatic Bisection**: Automatically identify which commit introduced a regression +5. **Integration with GitHub**: Post regression test results as PR comments diff --git a/eng/scripts/PrepareRepoForRegressionTesting.fsx b/eng/scripts/PrepareRepoForRegressionTesting.fsx new file mode 100644 index 0000000000..23494b5603 --- /dev/null +++ b/eng/scripts/PrepareRepoForRegressionTesting.fsx @@ -0,0 +1,87 @@ +/// Script to inject UseLocalCompiler.Directory.Build.props import into a third-party repository's Directory.Build.props +/// Usage: dotnet fsi PrepareRepoForRegressionTesting.fsx +/// +/// This script is designed to be run in the root of a third-party repository +/// It modifies the Directory.Build.props to import the UseLocalCompiler.Directory.Build.props + +open System +open System.IO +open System.Xml + +let propsFilePath = "Directory.Build.props" + +// Get the path to UseLocalCompiler.Directory.Build.props from command line args +let useLocalCompilerPropsPath = + let args = Environment.GetCommandLineArgs() + // When running with dotnet fsi, args are: [0]=dotnet; [1]=fsi.dll; [2]=script.fsx; [3...]=args + let scriptArgs = args |> Array.skipWhile (fun a -> not (a.EndsWith(".fsx"))) |> Array.skip 1 + if scriptArgs.Length > 0 then + scriptArgs.[0] + else + failwith "Usage: dotnet fsi PrepareRepoForRegressionTesting.fsx " + +printfn "PrepareRepoForRegressionTesting.fsx" +printfn "===================================" +printfn "UseLocalCompiler props path: %s" useLocalCompilerPropsPath + +// Verify the UseLocalCompiler props file exists +if not (File.Exists(useLocalCompilerPropsPath)) then + failwithf "UseLocalCompiler.Directory.Build.props not found at: %s" useLocalCompilerPropsPath + +printfn "✓ UseLocalCompiler.Directory.Build.props found" + +// Convert to absolute path and normalize slashes for MSBuild +let absolutePropsPath = + Path.GetFullPath(useLocalCompilerPropsPath).Replace("\\", "/") +printfn "Absolute path: %s" absolutePropsPath + +if File.Exists(propsFilePath) then + printfn "Directory.Build.props exists, modifying it..." + + // Load the existing XML + let doc = XmlDocument() + doc.PreserveWhitespace <- true + doc.Load(propsFilePath) + + // Find the Project element + let projectElement = doc.SelectSingleNode("/Project") + if isNull projectElement then + failwith "Could not find Project element in Directory.Build.props" + + // Check if our import already exists + let xpath = sprintf "//Import[contains(@Project, 'UseLocalCompiler.Directory.Build.props')]" + let existingImport = doc.SelectSingleNode(xpath) + + if isNull existingImport then + // Create Import element + let importElement = doc.CreateElement("Import") + importElement.SetAttribute("Project", absolutePropsPath) + + // Insert as first child of Project element + if projectElement.HasChildNodes then + projectElement.InsertBefore(importElement, projectElement.FirstChild) |> ignore + else + projectElement.AppendChild(importElement) |> ignore + + // Add newline for formatting + let newline = doc.CreateTextNode("\n ") + projectElement.InsertAfter(newline, importElement) |> ignore + + doc.Save(propsFilePath) + printfn "✓ Added UseLocalCompiler import to Directory.Build.props" + else + printfn "✓ UseLocalCompiler import already exists" +else + printfn "Directory.Build.props does not exist, creating it..." + let newContent = sprintf "\n \n\n" absolutePropsPath + File.WriteAllText(propsFilePath, newContent) + printfn "✓ Created Directory.Build.props with UseLocalCompiler import" + +// Print the final content +printfn "" +printfn "Final Directory.Build.props content:" +printfn "-----------------------------------" +let content = File.ReadAllText(propsFilePath) +printfn "%s" content +printfn "-----------------------------------" +printfn "✓ Repository prepared for regression testing" diff --git a/eng/templates/regression-test-jobs.yml b/eng/templates/regression-test-jobs.yml new file mode 100644 index 0000000000..449398a50f --- /dev/null +++ b/eng/templates/regression-test-jobs.yml @@ -0,0 +1,260 @@ +# Template for F# Compiler Regression Tests +# Tests third-party F# projects with the freshly built compiler + +parameters: +- name: testMatrix + type: object +- name: dependsOn + type: string + default: 'EndToEndBuildTests' + +jobs: +- ${{ each item in parameters.testMatrix }}: + - job: RegressionTest_${{ replace(item.displayName, '-', '_') }}_${{ replace(replace(item.repo, '/', '_'), '-', '_') }} + displayName: '${{ item.displayName }} Regression Test' + dependsOn: ${{ parameters.dependsOn }} + ${{ if item.useVmImage }}: + pool: + vmImage: ${{ item.useVmImage }} + ${{ else }}: + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(_WindowsMachineQueueName) + timeoutInMinutes: 60 + steps: + - checkout: self + displayName: Checkout F# compiler repo (for scripts) + + - task: DownloadPipelineArtifact@2 + displayName: Download F# Compiler FSC Artifacts + inputs: + artifactName: 'FSharpCompilerFscArtifacts' + downloadPath: '$(Pipeline.Workspace)/FSharpCompiler/artifacts/bin/fsc' + + - task: DownloadPipelineArtifact@2 + displayName: Download F# Core Artifacts + inputs: + artifactName: 'FSharpCoreArtifacts' + downloadPath: '$(Pipeline.Workspace)/FSharpCompiler/artifacts/bin/FSharp.Core' + + - task: DownloadPipelineArtifact@2 + displayName: Download UseLocalCompiler props + inputs: + artifactName: 'UseLocalCompilerProps' + downloadPath: '$(Pipeline.Workspace)/Props' + + - pwsh: | + Write-Host "Cloning repository: ${{ item.repo }}" + git clone --recursive https://github.com/${{ item.repo }}.git $(Pipeline.Workspace)/TestRepo + Set-Location $(Pipeline.Workspace)/TestRepo + + Write-Host "Checking out commit: ${{ item.commit }}" + git checkout ${{ item.commit }} + + Write-Host "Initializing submodules (if any)..." + git submodule update --init --recursive + + Write-Host "Successfully checked out ${{ item.repo }} at commit ${{ item.commit }}" + git log -1 --oneline + + Write-Host "Repository structure:" + Get-ChildItem -Name + + Write-Host "Verifying build script exists: ${{ item.buildScript }}" + if (Test-Path "${{ item.buildScript }}") { + Write-Host "Build script found: ${{ item.buildScript }}" + } else { + Write-Host "Build script not found: ${{ item.buildScript }}" + Write-Host "Available files in root:" + Get-ChildItem + exit 1 + } + displayName: Checkout ${{ item.displayName }} at specific commit + + - pwsh: | + Set-Location $(Pipeline.Workspace)/TestRepo + Write-Host "Removing global.json to use latest SDK..." + if (Test-Path "global.json") { + Remove-Item "global.json" -Force + Write-Host "global.json removed" + } else { + Write-Host "No global.json found" + } + displayName: Remove global.json to use latest SDK + + - task: UseDotNet@2 + displayName: Install .NET SDK 8.0.x for ${{ item.displayName }} + inputs: + packageType: sdk + version: '8.0.x' + installationPath: $(Pipeline.Workspace)/TestRepo/.dotnet + + - task: UseDotNet@2 + displayName: Install .NET SDK 10.0.100 for ${{ item.displayName }} + inputs: + packageType: sdk + version: '10.0.100' + installationPath: $(Pipeline.Workspace)/TestRepo/.dotnet + + - pwsh: | + Set-Location $(Pipeline.Workspace)/TestRepo + + Write-Host "Running PrepareRepoForRegressionTesting.fsx..." + dotnet fsi $(Build.SourcesDirectory)/eng/scripts/PrepareRepoForRegressionTesting.fsx "$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props" + + if ($LASTEXITCODE -ne 0) { + Write-Host "Failed to prepare repository for regression testing" + exit 1 + } + displayName: Setup local compiler configuration for ${{ item.displayName }} + + - pwsh: | + Set-Location $(Pipeline.Workspace)/TestRepo + Write-Host "===========================================" + Write-Host "Environment Information for ${{ item.displayName }}" + Write-Host "===========================================" + dotnet --info + Write-Host "" + Write-Host "MSBuild version:" + dotnet msbuild -version + Write-Host "" + Write-Host "F# Compiler artifacts available:" + Get-ChildItem "$(Pipeline.Workspace)/FSharpCompiler/bin/fsc/Release/net10.0" -Name -ErrorAction SilentlyContinue + Write-Host "" + Write-Host "F# Core available:" + if (Test-Path "$(Pipeline.Workspace)/FSharpCompiler/bin/FSharp.Core/Release/netstandard2.0/FSharp.Core.dll") { + Write-Host "FSharp.Core.dll found" + } else { + Write-Host "FSharp.Core.dll not found" + } + Write-Host "" + Write-Host "Directory.Build.props content:" + Get-Content "Directory.Build.props" + Write-Host "" + Write-Host "===========================================" + displayName: Report build environment for ${{ item.displayName }} + + - pwsh: | + Set-Location $(Pipeline.Workspace)/TestRepo + Write-Host "============================================" + Write-Host "Starting build for ${{ item.displayName }}" + Write-Host "Repository: ${{ item.repo }}" + Write-Host "Commit: ${{ item.commit }}" + Write-Host "Build Script: ${{ item.buildScript }}" + Write-Host "============================================" + Write-Host "" + + # Use dotnet pack with binary logging on Windows to generate binlog files + # On Linux, execute the build script directly + if ($IsWindows) { + Write-Host "Running: dotnet pack build.proj -bl:build.binlog" + dotnet pack build.proj -bl:build.binlog + } else { + Write-Host "Executing: ${{ item.buildScript }}" + chmod +x "${{ item.buildScript }}" + bash -c "./${{ item.buildScript }}" + } + $exitCode = $LASTEXITCODE + + Write-Host "" + Write-Host "============================================" + Write-Host "Build completed for ${{ item.displayName }}" + Write-Host "Exit code: $exitCode" + Write-Host "============================================" + + if ($exitCode -ne 0) { + exit $exitCode + } + displayName: Build ${{ item.displayName }} with local F# compiler + env: + LocalFSharpCompilerPath: $(Pipeline.Workspace)/FSharpCompiler + LoadLocalFSharpBuild: 'True' + LocalFSharpCompilerConfiguration: Release + timeoutInMinutes: 45 + + - pwsh: | + Set-Location $(Pipeline.Workspace)/TestRepo + $binlogDir = "$(Pipeline.Workspace)/BinaryLogs" + New-Item -ItemType Directory -Force -Path $binlogDir | Out-Null + + Write-Host "Collecting .binlog files..." + $binlogs = Get-ChildItem -Path "." -Filter "*.binlog" -Recurse -ErrorAction SilentlyContinue + if ($binlogs.Count -eq 0) { + Write-Host "No .binlog files found" + } else { + foreach ($binlog in $binlogs) { + Write-Host "Copying: $($binlog.FullName)" + Copy-Item $binlog.FullName -Destination $binlogDir + } + Write-Host "Collected $($binlogs.Count) .binlog files" + } + displayName: Collect Binary Logs + condition: always() + continueOnError: true + + - task: PublishPipelineArtifact@1 + displayName: Publish ${{ item.displayName }} Binary Logs + inputs: + targetPath: '$(Pipeline.Workspace)/BinaryLogs' + artifactName: '${{ item.displayName }}_BinaryLogs' + publishLocation: pipeline + condition: always() + continueOnError: true + + - pwsh: | + Set-Location $(Pipeline.Workspace)/TestRepo + Write-Host "" + Write-Host "============================================" + Write-Host "Regression test completed for ${{ item.displayName }}" + Write-Host "Repository: ${{ item.repo }}" + Write-Host "Commit: ${{ item.commit }}" + Write-Host "Build Script: ${{ item.buildScript }}" + if ($env:AGENT_JOBSTATUS -eq "Succeeded") { + Write-Host "Status: SUCCESS" + Write-Host "The ${{ item.displayName }} library builds successfully with the new F# compiler" + } else { + Write-Host "Status: FAILED" + Write-Host "The ${{ item.displayName }} library failed to build with the new F# compiler" + + # Build multiline error message with reproduction steps + $lines = @( + "Regression test FAILED for ${{ item.displayName }} (${{ item.repo }}@${{ item.commit }})", + "", + "LOCAL REPRODUCTION STEPS (from fsharp repo root):", + "==========================================", + "# 1. Build the F# compiler", + "./build.sh -c Release", + "", + "# 2. Clone and checkout the failing library", + "cd ..", + "git clone --recursive https://github.com/${{ item.repo }}.git TestRepo", + "cd TestRepo", + "git checkout ${{ item.commit }}", + "git submodule update --init --recursive", + "rm -f global.json", + "", + "# 3. Prepare the repo for local compiler", + "dotnet fsi ../fsharp/eng/scripts/PrepareRepoForRegressionTesting.fsx `"../fsharp/UseLocalCompiler.Directory.Build.props`"", + "", + "# 4. Build with local compiler", + "export LocalFSharpCompilerPath=`$PWD/../fsharp", + "export LoadLocalFSharpBuild=True", + "export LocalFSharpCompilerConfiguration=Release", + "./${{ item.buildScript }}", + "==========================================" + ) + + # Report using VSO error format - each line separately + foreach ($line in $lines) { + Write-Host "##[error]$line" + } + + # Also log as VSO issue for Azure DevOps integration + Write-Host "##vso[task.logissue type=error;sourcepath=azure-pipelines-PR.yml]Regression test failed: ${{ item.displayName }}" + } + Write-Host "============================================" + + Write-Host "Binary logs found:" + Get-ChildItem "*.binlog" -Recurse -ErrorAction SilentlyContinue | ForEach-Object { Write-Host $_.FullName } + displayName: Report ${{ item.displayName }} test result + condition: always()