.NET: CI Build time end to end improvement

.github/workflows/dotnet-build-and-test.yml

@@ -59,20 +59,20 @@ jobs:
         if: steps.filter.outputs.dotnet != 'true'
         run: echo "NOT dotnet file"
 
-  dotnet-build-and-test:
+  # Build the full solution (including samples) on all TFMs. No tests.
+  dotnet-build:
     needs: paths-filter
     if: needs.paths-filter.outputs.dotnetChanges == 'true'
     strategy:
       fail-fast: false
       matrix:
         include:
-          - { targetFramework: "net10.0", os: "ubuntu-latest", configuration: Release, integration-tests: true, environment: "integration" }
+          - { targetFramework: "net10.0", os: "ubuntu-latest", configuration: Release }
           - { targetFramework: "net9.0", os: "windows-latest", configuration: Debug }
           - { targetFramework: "net8.0", os: "ubuntu-latest", configuration: Release }
-          - { targetFramework: "net472", os: "windows-latest", configuration: Release, integration-tests: true, environment: "integration" }
+          - { targetFramework: "net472", os: "windows-latest", configuration: Release }
 
     runs-on: ${{ matrix.os }}
-    environment: ${{ matrix.environment }}
     steps:
       - uses: actions/checkout@v6
         with:
@@ -84,16 +84,6 @@ jobs:
               python
               workflow-samples
 
-      # Start Cosmos DB Emulator for all integration tests and only for unit tests when CosmosDB changes happened)
-      - name: Start Azure Cosmos DB Emulator
-        if: ${{ runner.os == 'Windows' && (needs.paths-filter.outputs.cosmosDbChanges == 'true' || (github.event_name != 'pull_request' && matrix.integration-tests)) }}
-        shell: pwsh
-        run: |
-          Write-Host "Launching Azure Cosmos DB Emulator"
-          Import-Module "$env:ProgramFiles\Azure Cosmos DB Emulator\PSModules\Microsoft.Azure.CosmosDB.Emulator"
-          Start-CosmosDbEmulator -NoUI -Key "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="
-          echo "COSMOSDB_EMULATOR_AVAILABLE=true" >> $env:GITHUB_ENV
-
       - name: Setup dotnet
         uses: actions/setup-dotnet@v5.1.0
         with:
@@ -140,25 +130,98 @@ jobs:
           popd
           rm -rf "$TEMP_DIR"
 
-      - name: Run Unit Tests
+  # Build src+tests only (no samples) for a single TFM and run tests.
+  dotnet-test:
+    needs: paths-filter
+    if: needs.paths-filter.outputs.dotnetChanges == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { targetFramework: "net10.0", os: "ubuntu-latest", configuration: Release, integration-tests: true, environment: "integration" }
+          - { targetFramework: "net472", os: "windows-latest", configuration: Release, integration-tests: true, environment: "integration" }
+
+    runs-on: ${{ matrix.os }}
+    environment: ${{ matrix.environment }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+            persist-credentials: false
+            sparse-checkout: |
+              .
+              .github
+              dotnet
+              python
+              workflow-samples
+
+      # Start Cosmos DB Emulator for all integration tests and only for unit tests when CosmosDB changes happened)
+      - name: Start Azure Cosmos DB Emulator
+        if: ${{ runner.os == 'Windows' && (needs.paths-filter.outputs.cosmosDbChanges == 'true' || (github.event_name != 'pull_request' && matrix.integration-tests)) }}
+        shell: pwsh
+        run: |
+          Write-Host "Launching Azure Cosmos DB Emulator"
+          Import-Module "$env:ProgramFiles\Azure Cosmos DB Emulator\PSModules\Microsoft.Azure.CosmosDB.Emulator"
+          Start-CosmosDbEmulator -NoUI -Key "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="
+          echo "COSMOSDB_EMULATOR_AVAILABLE=true" >> $env:GITHUB_ENV
+
+      - name: Setup dotnet
+        uses: actions/setup-dotnet@v5.1.0
+        with:
+          global-json-file: ${{ github.workspace }}/dotnet/global.json
+
+      - name: Generate test solution (no samples)
+        shell: pwsh
+        run: |
+          ./dotnet/eng/scripts/New-FilteredSolution.ps1 `
+            -Solution dotnet/agent-framework-dotnet.slnx `
+            -TargetFramework ${{ matrix.targetFramework }} `
+            -Configuration ${{ matrix.configuration }} `
+            -ExcludeSamples `
+            -OutputPath dotnet/filtered.slnx `
+            -Verbose
+
+      - name: Build src and tests
         shell: bash
+        run: dotnet build dotnet/filtered.slnx -c ${{ matrix.configuration }} -f ${{ matrix.targetFramework }} --warnaserror
+
+      - name: Generate test-type filtered solutions
+        shell: pwsh
         run: |
-          export UT_PROJECTS=$(find ./dotnet -type f -name "*.UnitTests.csproj" | tr '\n' ' ')
-          for project in $UT_PROJECTS; do
-            # Query the project's target frameworks using MSBuild with the current configuration
-            target_frameworks=$(dotnet msbuild $project -getProperty:TargetFrameworks -p:Configuration=${{ matrix.configuration }} -nologo 2>/dev/null | tr -d '\r')
-
-            # Check if the project supports the target framework
-            if [[ "$target_frameworks" == *"${{ matrix.targetFramework }}"* ]]; then
-              if [[ "${{ matrix.targetFramework }}" == "${{ env.COVERAGE_FRAMEWORK }}" ]]; then
-                dotnet test -f ${{ matrix.targetFramework }} -c ${{ matrix.configuration }} $project --no-build -v Normal --logger trx --collect:"XPlat Code Coverage" --results-directory:"TestResults/Coverage/" -- DataCollectionRunSettings.DataCollectors.DataCollector.Configuration.ExcludeByAttribute=GeneratedCodeAttribute,CompilerGeneratedAttribute,ExcludeFromCodeCoverageAttribute
-              else
-                dotnet test -f ${{ matrix.targetFramework }} -c ${{ matrix.configuration }} $project --no-build -v Normal --logger trx
-              fi
-            else
-              echo "Skipping $project - does not support target framework ${{ matrix.targetFramework }} (supports: $target_frameworks)"
-            fi
-          done
+          $commonArgs = @{
+            Solution       = "dotnet/filtered.slnx"
+            TargetFramework = "${{ matrix.targetFramework }}"
+            Configuration   = "${{ matrix.configuration }}"
+            Verbose         = $true
+          }
+          ./dotnet/eng/scripts/New-FilteredSolution.ps1 @commonArgs `
+            -TestProjectNameFilter "*UnitTests*" `
+            -OutputPath dotnet/filtered-unit.slnx
+          ./dotnet/eng/scripts/New-FilteredSolution.ps1 @commonArgs `
+            -TestProjectNameFilter "*IntegrationTests*" `
+            -OutputPath dotnet/filtered-integration.slnx
+
+      - name: Run Unit Tests
+        shell: pwsh
+        working-directory: dotnet
+        run: |
+          $coverageSettings = Join-Path $PWD "tests/coverage.runsettings"
+          $coverageArgs = @()
+          if ("${{ matrix.targetFramework }}" -eq "${{ env.COVERAGE_FRAMEWORK }}") {
+            $coverageArgs = @(
+              "--coverage",
+              "--coverage-output-format", "cobertura",
+              "--coverage-settings", $coverageSettings,
+              "--results-directory", "../TestResults/Coverage/"
+            )
+          }
+
+          dotnet test --solution ./filtered-unit.slnx `
+            -f ${{ matrix.targetFramework }} `
+            -c ${{ matrix.configuration }} `
+            --no-build -v Normal `
+            --report-xunit-trx `
+            --ignore-exit-code 8 `
+            @coverageArgs
         env:
           # Cosmos DB Emulator connection settings
           COSMOSDB_ENDPOINT: https://localhost:8081
@@ -185,21 +248,19 @@ jobs:
         id: azure-functions-setup
 
       - name: Run Integration Tests
-        shell: bash
+        shell: pwsh
+        working-directory: dotnet
         if: github.event_name != 'pull_request' && matrix.integration-tests
         run: |
-          export INTEGRATION_TEST_PROJECTS=$(find ./dotnet -type f -name "*IntegrationTests.csproj" | tr '\n' ' ')
-          for project in $INTEGRATION_TEST_PROJECTS; do
-            # Query the project's target frameworks using MSBuild with the current configuration
-            target_frameworks=$(dotnet msbuild $project -getProperty:TargetFrameworks -p:Configuration=${{ matrix.configuration }} -nologo 2>/dev/null | tr -d '\r')
-
-            # Check if the project supports the target framework
-            if [[ "$target_frameworks" == *"${{ matrix.targetFramework }}"* ]]; then
-              dotnet test -f ${{ matrix.targetFramework }} -c ${{ matrix.configuration }} $project --no-build -v Normal --logger trx --filter "Category!=IntegrationDisabled"
-            else
-              echo "Skipping $project - does not support target framework ${{ matrix.targetFramework }} (supports: $target_frameworks)"
-            fi
-          done
+          dotnet test --solution ./filtered-integration.slnx `
+            -f ${{ matrix.targetFramework }} `
+            -c ${{ matrix.configuration }} `
+            --no-build -v Normal `
+            --report-xunit-trx `
+            --ignore-exit-code 8 `
+            --filter-not-trait "Category=IntegrationDisabled" `
+            --parallel-algorithm aggressive `
+            --max-threads 2.0x
         env:
           # Cosmos DB Emulator connection settings
           COSMOSDB_ENDPOINT: https://localhost:8081
@@ -222,7 +283,7 @@ jobs:
         if: matrix.targetFramework == env.COVERAGE_FRAMEWORK
         uses: danielpalme/ReportGenerator-GitHub-Action@5.5.1
         with:
-          reports: "./TestResults/Coverage/**/coverage.cobertura.xml"
+          reports: "./TestResults/Coverage/**/*.cobertura.xml"
           targetdir: "./TestResults/Reports"
           reporttypes: "HtmlInline;JsonSummary"
 
@@ -236,13 +297,13 @@ jobs:
       - name: Check coverage
         if: matrix.targetFramework == env.COVERAGE_FRAMEWORK
         shell: pwsh
-        run: .github/workflows/dotnet-check-coverage.ps1 -JsonReportPath "TestResults/Reports/Summary.json" -CoverageThreshold $env:COVERAGE_THRESHOLD
+        run: ./dotnet/eng/scripts/dotnet-check-coverage.ps1 -JsonReportPath "TestResults/Reports/Summary.json" -CoverageThreshold $env:COVERAGE_THRESHOLD
 
   # This final job is required to satisfy the merge queue. It must only run (or succeed) if no tests failed
   dotnet-build-and-test-check:
     if: always()
     runs-on: ubuntu-latest
-    needs: [dotnet-build-and-test]
+    needs: [dotnet-build, dotnet-test]
     steps:
       - name: Get Date
         shell: bash

dotnet/.github/skills/build-and-test/SKILL.md

@@ -17,14 +17,17 @@ dotnet format             # Auto-fix formatting for all projects
 
 # Build/test/format a specific project (preferred for isolated/internal changes)
 dotnet build src/Microsoft.Agents.AI.<Package> --tl:off
-dotnet test tests/Microsoft.Agents.AI.<Package>.UnitTests
+dotnet test --project tests/Microsoft.Agents.AI.<Package>.UnitTests
 dotnet format src/Microsoft.Agents.AI.<Package>
 
 # Run a single test
-dotnet test --filter "FullyQualifiedName~Namespace.TestClassName.TestMethodName"
+# Replace the filter values with the appropriate assembly, namespace, class, and method names for the test you want to run and use * as a wildcard elsewhere, e.g. "/*/*/HttpClientTests/GetAsync_ReturnsSuccessStatusCode"
+# Use `--ignore-exit-code 8` to avoid failing the build when no tests are found for some projects
+dotnet test --filter-query "/<assemblyFilter>/<namespaceFilter>/<classFilter>/<methodFilter>" --ignore-exit-code 8
 
 # Run unit tests only
-dotnet test --filter FullyQualifiedName\~UnitTests
+# Use `--ignore-exit-code 8` to avoid failing the build when no tests are found for integration test projects
+dotnet test --filter-query "/*UnitTests*/*/*/*" --ignore-exit-code 8
 ```
 
 Use `--tl:off` when building to avoid flickering when running commands in the agent.
@@ -56,15 +59,15 @@ Example: Running tests for a single project using .NET 10.
 
 ```bash
 # From dotnet/ directory
-dotnet test ./tests/Microsoft.Agents.AI.Abstractions.UnitTests -f net10.0
+dotnet test --project ./tests/Microsoft.Agents.AI.Abstractions.UnitTests -f net10.0
 ```
 
 Example: Running a single test in a specific project using .NET 10.
 Provide the full namespace, class name, and method name for the test you want to run:
 
 ```bash
 # From dotnet/ directory
-dotnet test ./tests/Microsoft.Agents.AI.Abstractions.UnitTests -f net10.0 --filter "FullyQualifiedName~Microsoft.Agents.AI.Abstractions.UnitTests.AgentRunOptionsTests.CloningConstructorCopiesProperties"
+dotnet test --project ./tests/Microsoft.Agents.AI.Abstractions.UnitTests -f net10.0 --filter-query "/*/Microsoft.Agents.AI.Abstractions.UnitTests/AgentRunOptionsTests/CloningConstructorCopiesProperties"
 ```
 
 ### Multi-target framework tip
@@ -83,3 +86,45 @@ Just remember to run `dotnet restore` after pulling changes, making changes to p
 Unit tests target both .NET Framework as well as .NET Core. When running on Linux, only the .NET Core tests can be run, as .NET Framework is not supported on Linux.
 
 To run only the .NET Core tests, use the `-f net10.0` option with `dotnet test`.
+
+### Microsoft Testing Platform (MTP)
+
+Tests use the [Microsoft Testing Platform](https://learn.microsoft.com/dotnet/core/testing/unit-testing-platform-intro) via xUnit v3. Key differences from the legacy VSTest runner:
+
+- **`dotnet test` requires `--project`** to specify a test project directly (positional arguments are no longer supported).
+- **Test output** uses the MTP format (e.g., `[✓112/x0/↓0]` progress and `Test run summary: Passed!`).
+- **TRX reports** use `--report-xunit-trx` instead of `--logger trx`.
+- **Code coverage** uses `Microsoft.Testing.Extensions.CodeCoverage` with `--coverage --coverage-output-format cobertura`.
+- **Running a test project directly** is supported via `dotnet run --project <test-project>`. This bypasses the `dotnet test` infrastructure and runs the test executable directly with the MTP command line.
+
+- **Running tests across the solution** with a filter may cause some projects to match zero tests, which MTP treats as a failure (exit code 8). Use `--ignore-exit-code 8` to suppress this:
+
+```bash
+# Run all unit tests across the solution, ignoring projects with no matching tests
+dotnet test --solution ./agent-framework-dotnet.slnx --no-build -f net10.0 --ignore-exit-code 8
+```
+
+- **Running tests with `--solution` for a specific TFM** requires all projects in the solution to support that TFM. Not all projects target every framework (e.g., some are `net10.0`-only). Use `./dotnet/eng/scripts/New-FilteredSolution.ps1` to generate a filtered solution:
+
+```powershell
+# Generate a filtered solution for net472 and run tests
+$filtered = ./dotnet/eng/scripts/New-FilteredSolution.ps1 -Solution dotnet/agent-framework-dotnet.slnx -TargetFramework net472
+dotnet test --solution $filtered --no-build -f net472 --ignore-exit-code 8
+
+# Exclude samples and keep only unit test projects
+./dotnet/eng/scripts/New-FilteredSolution.ps1 -Solution dotnet/agent-framework-dotnet.slnx -TargetFramework net10.0 -ExcludeSamples -TestProjectNameFilter "*UnitTests*" -OutputPath dotnet/filtered-unit.slnx
+```
+
+```bash
+# Run tests via dotnet test (uses MTP under the hood)
+dotnet test --project ./tests/Microsoft.Agents.AI.UnitTests -f net10.0
+
+# Run tests with code coverage (Cobertura format)
+dotnet test --project ./tests/Microsoft.Agents.AI.UnitTests -f net10.0 --coverage --coverage-output-format cobertura --coverage-settings ./tests/coverage.runsettings
+
+# Run tests directly via dotnet run (MTP native command line)
+dotnet run --project ./tests/Microsoft.Agents.AI.UnitTests -f net10.0
+
+# Show MTP command line help
+dotnet run --project ./tests/Microsoft.Agents.AI.UnitTests -f net10.0 -- -?
+```

dotnet/Directory.Packages.props

@@ -138,12 +138,10 @@
     <PackageVersion Include="Microsoft.AspNetCore.TestHost" Condition="'$(TargetFramework)' == 'net10.0'" Version="10.0.0" />
     <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.0.0" />
     <PackageVersion Include="Moq" Version="[4.18.4]" />
-    <PackageVersion Include="xunit" Version="2.9.3" />
-    <PackageVersion Include="xunit.abstractions" Version="2.0.3" />
-    <PackageVersion Include="xunit.runner.visualstudio" Version="3.1.3" />
-    <PackageVersion Include="Xunit.SkippableFact" Version="1.5.23" />
-    <PackageVersion Include="xretry" Version="1.9.0" />
-    <PackageVersion Include="coverlet.collector" Version="6.0.4" />
+    <PackageVersion Include="xunit.v3.mtp-v2" Version="3.2.2" />
+    <PackageVersion Include="xunit.runner.visualstudio" Version="3.1.5" />
+    <PackageVersion Include="xRetry.v3" Version="1.0.0-rc3" />
+    <PackageVersion Include="Microsoft.Testing.Extensions.CodeCoverage" Version="18.4.1" />
     <!-- Symbols -->
     <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
     <!-- Toolset -->

dotnet/agent-framework-dotnet.slnx

@@ -305,7 +305,6 @@
   </Folder>
   <Folder Name="/Solution Items/.github/workflows/">
     <File Path="../.github/workflows/dotnet-build-and-test.yml" />
-    <File Path="../.github/workflows/dotnet-check-coverage.ps1" />
     <File Path="../.github/workflows/dotnet-format.yml" />
   </Folder>
   <Folder Name="/Solution Items/demos/">
@@ -338,6 +337,10 @@
     <File Path="eng/MSBuild/Shared.props" />
     <File Path="eng/MSBuild/Shared.targets" />
   </Folder>
+  <Folder Name="/Solution Items/eng/scripts/">
+    <File Path="eng/scripts/dotnet-check-coverage.ps1" />
+    <File Path="eng/scripts/New-FilteredSolution.ps1" />
+  </Folder>
   <Folder Name="/Solution Items/nuget/">
     <File Path="nuget/icon.png" />
     <File Path="nuget/nuget-package.props" />