From 9c21329f7d9951189ffaeb50bcd635824df0cc8f Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Sat, 9 May 2026 17:27:48 +0200 Subject: [PATCH 1/8] Add Android BenchmarkDotNet template Adds an Android BenchmarkDotNet project template that runs benchmarks through an instrumentation class on CoreCLR. The template uses BenchmarkDotNet's in-process no-emit toolchain and supports dotnet run argument pass-through for common options such as --filter and --job. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/Versions.props | 1 + src/Microsoft.Android.Run/Program.cs | 141 ++++++++++++++++-- .../.template.config/template.json | 34 +++++ .../AndroidBenchmark1.csproj | 24 +++ .../AndroidManifest.xml | 5 + .../BenchmarkInstrumentation.cs | 136 +++++++++++++++++ .../StringBenchmarks.cs | 18 +++ .../Microsoft.Android.Sdk.Application.targets | 3 +- .../Xamarin.ProjectTools/Common/DotNetCLI.cs | 20 ++- .../MSBuildDeviceIntegration.csproj | 4 + .../Tests/InstallAndRunTests.cs | 79 ++++++++++ 11 files changed, 451 insertions(+), 14 deletions(-) create mode 100644 src/Microsoft.Android.Templates/android-benchmarkdotnet/.template.config/template.json create mode 100644 src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidBenchmark1.csproj create mode 100644 src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidManifest.xml create mode 100644 src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs create mode 100644 src/Microsoft.Android.Templates/android-benchmarkdotnet/StringBenchmarks.cs diff --git a/eng/Versions.props b/eng/Versions.props index 2dfce5d2459..8568fccf4b5 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -15,6 +15,7 @@ $(MicrosoftNETWorkloadEmscriptenCurrentManifest110100preview4PackageVersion) 11.0.100-preview.5.26251.112 0.11.5-preview.26251.112 + 0.15.8 4.3.0-preview.26252.2 10.0.7 11.0.0-preview.1.26104.118 diff --git a/src/Microsoft.Android.Run/Program.cs b/src/Microsoft.Android.Run/Program.cs index c7f9cac47e8..8ae2e28d475 100644 --- a/src/Microsoft.Android.Run/Program.cs +++ b/src/Microsoft.Android.Run/Program.cs @@ -1,9 +1,11 @@ using System.Diagnostics; +using System.Text; using Mono.Options; using Xamarin.Android.Tools; const string Name = "Microsoft.Android.Run"; const string VersionsFileName = "Microsoft.Android.versions.txt"; +const string BenchmarkArgsKey = "benchmarkArgsBase64"; string? adbPath = null; string? adbTarget = null; @@ -32,6 +34,7 @@ async Task RunAsync (string[] args) { bool showHelp = false; bool showVersion = false; + var remaining = new List (); var options = new OptionSet { $"Usage: {Name} [OPTIONS]", @@ -77,18 +80,21 @@ async Task RunAsync (string[] args) { "h|help|?", "Show this help message and exit.", v => showHelp = v != null }, + { "<>", + v => remaining.Add (v) }, }; - List remaining; try { - remaining = options.Parse (args); + options.Parse (args); } catch (OptionException e) { Console.Error.WriteLine ($"Error: {e.Message}"); Console.Error.WriteLine ($"Try '{Name} --help' for more information."); return 1; } - if (remaining.Count > 0 && !isDotnetTestMode) { + bool isInstrumentMode = !string.IsNullOrEmpty (instrumentation); + + if (remaining.Count > 0 && !isDotnetTestMode && !isInstrumentMode) { Console.Error.WriteLine ($"Error: Unexpected argument(s): {string.Join (" ", remaining)}"); Console.Error.WriteLine ($"Try '{Name} --help' for more information."); return 1; @@ -124,8 +130,6 @@ async Task RunAsync (string[] args) return 1; } - bool isInstrumentMode = !string.IsNullOrEmpty (instrumentation); - if (!isInstrumentMode && string.IsNullOrEmpty (activity) && !isDotnetTestMode) { Console.Error.WriteLine ("Error: --activity or --instrument is required."); Console.Error.WriteLine ($"Try '{Name} --help' for more information."); @@ -181,7 +185,7 @@ async Task RunAsync (string[] args) return await RunDotnetTestAsync (remaining); if (isInstrumentMode) - return await RunInstrumentationAsync (); + return await RunInstrumentationAsync (remaining); return await RunAppAsync (); } finally { @@ -212,11 +216,13 @@ void OnCancelKeyPress (object? sender, ConsoleCancelEventArgs e) } } -async Task RunInstrumentationAsync () +async Task RunInstrumentationAsync (List instrumentationArguments) { // Build the am instrument command var userArg = string.IsNullOrEmpty (deviceUserId) ? "" : $" --user {deviceUserId}"; - var cmdArgs = $"shell am instrument -w{userArg} {package}/{instrumentation}"; + var benchmarkArgs = EncodeBenchmarkArguments (instrumentationArguments); + var benchmarkArg = benchmarkArgs == null ? "" : $" -e {BenchmarkArgsKey} {benchmarkArgs}"; + var cmdArgs = $"shell am instrument -w{userArg}{benchmarkArg} {package}/{instrumentation}"; if (verbose) Console.WriteLine ($"Running instrumentation: adb {cmdArgs}"); @@ -226,17 +232,25 @@ async Task RunInstrumentationAsync () using var instrumentProcess = new Process { StartInfo = psi }; var locker = new Lock (); + var output = new StringBuilder (); + var error = new StringBuilder (); instrumentProcess.OutputDataReceived += (s, e) => { - if (e.Data != null) - lock (locker) + if (e.Data != null) { + lock (locker) { + output.AppendLine (e.Data); Console.WriteLine (e.Data); + } + } }; instrumentProcess.ErrorDataReceived += (s, e) => { - if (e.Data != null) - lock (locker) + if (e.Data != null) { + lock (locker) { + error.AppendLine (e.Data); Console.Error.WriteLine (e.Data); + } + } }; instrumentProcess.Start (); @@ -278,9 +292,98 @@ async Task RunInstrumentationAsync () return 1; } + var result = ParseInstrumentationRunOutput (output.ToString (), error.ToString ()); + if (!string.IsNullOrEmpty (result.Summary)) + Console.WriteLine (result.Summary); + if (!string.IsNullOrEmpty (result.ArtifactsPath)) { + var localArtifactsPath = await PullArtifactsAsync (result.ArtifactsPath); + if (!string.IsNullOrEmpty (localArtifactsPath)) + Console.WriteLine ($"BenchmarkDotNet artifacts: {localArtifactsPath}"); + } + + if (!result.Succeeded) { + if (!string.IsNullOrEmpty (result.Error)) + Console.Error.WriteLine (result.Error); + else if (result.InstrumentationCode.HasValue) + Console.Error.WriteLine ($"Instrumentation failed with code {result.InstrumentationCode.Value}."); + else + Console.Error.WriteLine ("Instrumentation failed."); + return 1; + } + return 0; } +static string? EncodeBenchmarkArguments (List arguments) +{ + if (arguments.Count == 0) + return null; + + var joinedArguments = string.Join ('\0', arguments); + return Convert.ToBase64String (Encoding.UTF8.GetBytes (joinedArguments)); +} + +InstrumentationRunResult ParseInstrumentationRunOutput (string output, string error) +{ + var result = new InstrumentationRunResult (); + foreach (var rawLine in (output + "\n" + error).Split ('\n')) { + var line = rawLine.TrimEnd ('\r'); + if (line.StartsWith ("INSTRUMENTATION_RESULT: ", StringComparison.Ordinal)) { + var value = line.Substring ("INSTRUMENTATION_RESULT: ".Length); + var separator = value.IndexOf ('='); + if (separator <= 0) + continue; + + var key = value.Substring (0, separator).Trim (); + var keyValue = value.Substring (separator + 1).Trim (); + switch (key) { + case "artifactsPath": + result.ArtifactsPath = keyValue; + break; + case "summary": + result.Summary = keyValue; + break; + case "error": + result.Error = keyValue; + break; + } + } else if (line.StartsWith ("INSTRUMENTATION_CODE: ", StringComparison.Ordinal)) { + var code = line.Substring ("INSTRUMENTATION_CODE: ".Length).Trim (); + if (int.TryParse (code, out int instrumentationCode)) + result.InstrumentationCode = instrumentationCode; + } else if (line.StartsWith ("INSTRUMENTATION_FAILED", StringComparison.Ordinal)) { + result.InstrumentationFailed = true; + if (string.IsNullOrEmpty (result.Error)) + result.Error = line; + } + } + + return result; +} + +async Task PullArtifactsAsync (string deviceArtifactsPath) +{ + var localArtifactsPath = Path.Combine (Environment.CurrentDirectory, Path.GetFileName (deviceArtifactsPath.TrimEnd ('/'))); + + if (verbose) + Console.WriteLine ($"Pulling BenchmarkDotNet artifacts: {deviceArtifactsPath} -> {localArtifactsPath}"); + + var (exitCode, output, error) = await AdbHelper.RunAsync ( + adbPath, + adbTarget, + $"pull \"{deviceArtifactsPath}\" \"{Environment.CurrentDirectory}\"", + cts.Token, + verbose); + if (exitCode != 0) { + Console.Error.WriteLine ($"Error: Failed to pull BenchmarkDotNet artifacts: {error}"); + if (verbose && !string.IsNullOrWhiteSpace (output)) + Console.Error.WriteLine (output); + return null; + } + + return localArtifactsPath; +} + async Task RunDotnetTestAsync (List mtpArgs) { if (verbose) @@ -520,3 +623,17 @@ async Task StopAppAsync () return (null, null); } } + +class InstrumentationRunResult +{ + public string? ArtifactsPath { get; set; } + public string? Summary { get; set; } + public string? Error { get; set; } + public int? InstrumentationCode { get; set; } + public bool InstrumentationFailed { get; set; } + + public bool Succeeded => + !InstrumentationFailed && + string.IsNullOrEmpty (Error) && + (!InstrumentationCode.HasValue || InstrumentationCode.Value == -1); +} diff --git a/src/Microsoft.Android.Templates/android-benchmarkdotnet/.template.config/template.json b/src/Microsoft.Android.Templates/android-benchmarkdotnet/.template.config/template.json new file mode 100644 index 00000000000..8d121d4300e --- /dev/null +++ b/src/Microsoft.Android.Templates/android-benchmarkdotnet/.template.config/template.json @@ -0,0 +1,34 @@ +{ + "$schema": "http://json.schemastore.org/template", + "author": "Microsoft", + "classifications": [ "Android", "Mobile", "Benchmark" ], + "identity": "Microsoft.Android.AndroidBenchmarkDotNet", + "name": "Android BenchmarkDotNet Project", + "description": "A project for creating a .NET for Android BenchmarkDotNet project", + "shortName": "android-benchmarkdotnet", + "tags": { + "language": "C#", + "type": "project" + }, + "sourceName": "AndroidBenchmark1", + "preferNameDirectory": true, + "primaryOutputs": [ + { "path": "AndroidBenchmark1.csproj" } + ], + "symbols": { + "packageName": { + "type": "parameter", + "description": "Overrides the package name in the AndroidManifest.xml", + "datatype": "string", + "replaces": "com.companyname.AndroidBenchmark1" + }, + "supportedOSVersion": { + "type": "parameter", + "description": "Overrides $(SupportedOSPlatformVersion) in the project", + "datatype": "string", + "replaces": "SUPPORTED_OS_PLATFORM_VERSION", + "defaultValue": "24" + } + }, + "defaultName": "AndroidBenchmark1" +} diff --git a/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidBenchmark1.csproj b/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidBenchmark1.csproj new file mode 100644 index 00000000000..758dc720396 --- /dev/null +++ b/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidBenchmark1.csproj @@ -0,0 +1,24 @@ + + + net11.0-android + SUPPORTED_OS_PLATFORM_VERSION + AndroidBenchmark1 + Exe + enable + none + enable + com.companyname.AndroidBenchmark1 + 1 + 1.0 + false + android-arm64;android-x64 + apk + com.companyname.AndroidBenchmark1.BenchmarkInstrumentation + false + false + + + + + + diff --git a/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidManifest.xml b/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidManifest.xml new file mode 100644 index 00000000000..7c071193952 --- /dev/null +++ b/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidManifest.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs b/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs new file mode 100644 index 00000000000..8416e7a040d --- /dev/null +++ b/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs @@ -0,0 +1,136 @@ +using System.Text; +using Android.App; +using Android.OS; +using Android.Runtime; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Filters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.InProcess.NoEmit; + +namespace AndroidBenchmark1; + +[Instrumentation (Name = "com.companyname.AndroidBenchmark1.BenchmarkInstrumentation")] +public class BenchmarkInstrumentation : Instrumentation +{ + const string BenchmarkArgsKey = "benchmarkArgsBase64"; + + Bundle? arguments; + + protected BenchmarkInstrumentation (IntPtr handle, JniHandleOwnership ownership) + : base (handle, ownership) + { + } + + public override void OnCreate (Bundle? arguments) + { + base.OnCreate (arguments); + this.arguments = arguments; + Start (); + } + + public override void OnStart () + { + base.OnStart (); + + Task.Run (() => { + var bundle = new Bundle (); + try { + var writablePath = Application.Context.GetExternalFilesDir (null)?.AbsolutePath ?? Path.GetTempPath (); + var artifactsPath = Path.Combine (writablePath, "BenchmarkDotNet.Artifacts"); + Directory.CreateDirectory (artifactsPath); + + var benchmarkArgs = DecodeBenchmarkArguments (arguments); + var summaries = RunBenchmarks (benchmarkArgs, artifactsPath); + var benchmarkCount = summaries.Sum (summary => summary.Reports.Length); + var failed = summaries.Length == 0 || summaries.Any (summary => summary.HasCriticalValidationErrors || summary.Reports.Any (report => !report.Success)); + + bundle.PutInt ("benchmarks", benchmarkCount); + bundle.PutString ("artifactsPath", artifactsPath); + bundle.PutString ("summary", $"BenchmarkDotNet completed {benchmarkCount} benchmark report(s)."); + Finish (failed ? Result.Canceled : Result.Ok, bundle); + } catch (Exception ex) { + bundle.PutString ("error", ex.ToString ()); + Finish (Result.Canceled, bundle); + } + }); + } + + static string [] DecodeBenchmarkArguments (Bundle? arguments) + { + var encodedArgs = arguments?.GetString (BenchmarkArgsKey); + if (string.IsNullOrEmpty (encodedArgs)) + return []; + + try { + var decodedArgs = Encoding.UTF8.GetString (Convert.FromBase64String (encodedArgs)); + return decodedArgs.Length == 0 ? [] : decodedArgs.Split ('\0'); + } catch (FormatException ex) { + throw new InvalidOperationException ($"Invalid BenchmarkDotNet argument payload in '{BenchmarkArgsKey}'.", ex); + } + } + + static Summary [] RunBenchmarks (string [] benchmarkArgs, string artifactsPath) + { + var logger = ConsoleLogger.Default; + var options = ParseBenchmarkArguments (benchmarkArgs); + var config = DefaultConfig.Instance + .AddJob (options.Job.WithToolchain (InProcessNoEmitToolchain.Instance)) + .WithArtifactsPath (artifactsPath); + if (options.Filters.Count > 0) + config = config.AddFilter (new GlobFilter (options.Filters.ToArray ())); + + var (allTypesValid, runnableTypes) = TypeFilter.GetTypesWithRunnableBenchmarks ([], [typeof (BenchmarkInstrumentation).Assembly], logger); + if (!allTypesValid || runnableTypes.Count == 0) + return []; + + var benchmarkRunInfos = TypeFilter.Filter (config, runnableTypes); + return BenchmarkRunner.Run (benchmarkRunInfos); + } + + static BenchmarkOptions ParseBenchmarkArguments (string [] benchmarkArgs) + { + var options = new BenchmarkOptions (); + for (int i = 0; i < benchmarkArgs.Length; i++) { + var argument = benchmarkArgs [i]; + if (argument == "-f" || argument == "--filter") { + options.Filters.Add (GetRequiredArgumentValue (benchmarkArgs, ref i, argument)); + } else if (argument.StartsWith ("--filter=", StringComparison.Ordinal)) { + options.Filters.Add (argument.Substring ("--filter=".Length)); + } else if (argument == "-j" || argument == "--job") { + options.Job = GetJob (GetRequiredArgumentValue (benchmarkArgs, ref i, argument)); + } else if (argument.StartsWith ("--job=", StringComparison.Ordinal)) { + options.Job = GetJob (argument.Substring ("--job=".Length)); + } else { + throw new NotSupportedException ($"Unsupported BenchmarkDotNet argument '{argument}'. The Android BenchmarkDotNet template currently supports --filter/-f and --job/-j."); + } + } + + return options; + } + + static string GetRequiredArgumentValue (string [] benchmarkArgs, ref int index, string argument) + { + if (index + 1 >= benchmarkArgs.Length) + throw new ArgumentException ($"Missing value for '{argument}'."); + return benchmarkArgs [++index]; + } + + static Job GetJob (string value) => + value.ToLowerInvariant () switch { + "default" => Job.Default, + "dry" => Job.Dry, + "short" => Job.ShortRun, + "medium" => Job.MediumRun, + "long" => Job.LongRun, + _ => throw new ArgumentException ($"Unsupported BenchmarkDotNet job '{value}'. Supported jobs are Default, Dry, Short, Medium, and Long."), + }; + + sealed class BenchmarkOptions + { + public Job Job { get; set; } = Job.Default; + public List Filters { get; } = []; + } +} diff --git a/src/Microsoft.Android.Templates/android-benchmarkdotnet/StringBenchmarks.cs b/src/Microsoft.Android.Templates/android-benchmarkdotnet/StringBenchmarks.cs new file mode 100644 index 00000000000..93a4a17ca53 --- /dev/null +++ b/src/Microsoft.Android.Templates/android-benchmarkdotnet/StringBenchmarks.cs @@ -0,0 +1,18 @@ +using BenchmarkDotNet.Attributes; + +namespace AndroidBenchmark1; + +[MemoryDiagnoser] +public class StringBenchmarks +{ + readonly string [] values = Enumerable.Range (0, 100).Select (i => i.ToString ()).ToArray (); + + [Benchmark] + public int ParseIntegers () + { + var sum = 0; + foreach (var value in values) + sum += int.Parse (value); + return sum; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Application.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Application.targets index 03d9295a4f8..9bcd92289c8 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Application.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Application.targets @@ -127,8 +127,9 @@ This file contains targets specific for Android application projects. <_AndroidRunUserArg Condition=" '$(AndroidDeviceUserId)' != '' ">--user "$(AndroidDeviceUserId)" <_AndroidRunInstrumentArg Condition=" '$(AndroidInstrumentation)' != '' ">--instrument "$(AndroidInstrumentation)" <_AndroidRunActivityArg Condition=" '$(AndroidInstrumentation)' == '' ">--activity "$(AndroidLaunchActivity)" + <_AndroidRunStartArguments Condition=" '$(AndroidInstrumentation)' != '' and '$(StartArguments)' != '' ">$(StartArguments) dotnet - exec "$(_AndroidRunPath)" --adb "$(_AdbToolPath)" $(_AndroidRunAdbTargetArg) --package "$(_AndroidPackage)" $(_AndroidRunActivityArg) $(_AndroidRunInstrumentArg) --logcat-args "$(_AndroidRunLogcatArgs)" $(_AndroidRunUserArg) $(_AndroidRunExtraArgs) + exec "$(_AndroidRunPath)" --adb "$(_AdbToolPath)" $(_AndroidRunAdbTargetArg) --package "$(_AndroidPackage)" $(_AndroidRunActivityArg) $(_AndroidRunInstrumentArg) --logcat-args "$(_AndroidRunLogcatArgs)" $(_AndroidRunUserArg) $(_AndroidRunExtraArgs) $(_AndroidRunStartArguments) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs index 47e5b4f90d1..9e8fcbed405 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs @@ -105,13 +105,31 @@ protected bool Execute (params string [] args) return succeeded; } - public bool New (string template, string output = null) + public bool New (string template, string output = null, string customHive = null) { var arguments = new List { "new", template, "--output", $"\"{output ?? ProjectDirectory}\"", }; + if (!string.IsNullOrEmpty (customHive)) { + arguments.Add ("--debug:custom-hive"); + arguments.Add ($"\"{customHive}\""); + } + return Execute (arguments.ToArray ()); + } + + public bool NewInstall (string templateSource, string customHive = null) + { + var arguments = new List { + "new", + "install", + $"\"{templateSource}\"", + }; + if (!string.IsNullOrEmpty (customHive)) { + arguments.Add ("--debug:custom-hive"); + arguments.Add ($"\"{customHive}\""); + } return Execute (arguments.ToArray ()); } diff --git a/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj b/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj index 051786dd116..baeeeedaad6 100644 --- a/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj +++ b/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj @@ -50,6 +50,10 @@ <_Parameter1>MSTestPackageVersion <_Parameter2>$(MSTestPackageVersion) + + <_Parameter1>BenchmarkDotNetPackageVersion + <_Parameter2>$(BenchmarkDotNetPackageVersion) + diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs index f178cda5a5a..8363594d372 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs @@ -2332,6 +2332,85 @@ public void DotNetNewAndroidTest (string mode, AndroidRuntime runtime) } } + [Test] + public void DotNetNewAndroidBenchmarkDotNet () + { + const string templateName = "DotNetNewAndroidBenchmarkDotNet"; + var projectDirectory = Path.Combine (Root, "temp", templateName); + if (Directory.Exists (projectDirectory)) + Directory.Delete (projectDirectory, true); + + TestOutputDirectories [TestContext.CurrentContext.Test.ID] = projectDirectory; + var dotnet = new DotNetCLI (Path.Combine (projectDirectory, $"{templateName}.csproj")); + var templateHive = Path.Combine (projectDirectory, ".template-hive"); + var templateSource = Path.Combine (XABuildPaths.TopDirectory, "src", "Microsoft.Android.Templates"); + Assert.IsTrue (dotnet.NewInstall (templateSource, templateHive), $"`dotnet new install {templateSource}` should succeed"); + Assert.IsTrue (dotnet.New ("android-benchmarkdotnet", customHive: templateHive), "`dotnet new android-benchmarkdotnet` should succeed"); + + var benchmarkDotNetVersion = GetAssemblyMetadataValue ("BenchmarkDotNetPackageVersion"); + var csprojPath = Path.Combine (projectDirectory, $"{templateName}.csproj"); + var doc = XDocument.Load (csprojPath); + var ns = doc.Root?.Name.Namespace ?? XNamespace.None; + var benchmarkDotNetRef = doc.Descendants (ns + "PackageReference") + .FirstOrDefault (e => e.Attribute ("Include")?.Value == "BenchmarkDotNet"); + Assert.IsNotNull (benchmarkDotNetRef, "BenchmarkDotNet PackageReference should exist in the generated project"); + benchmarkDotNetRef.SetAttributeValue ("Version", benchmarkDotNetVersion); + doc.Save (csprojPath); + + var buildParameters = new List { + "Configuration=Release", + "UseMonoRuntime=false", + "RestoreAdditionalProjectSources=https://api.nuget.org/v3/index.json", + }; + Assert.IsTrue (dotnet.Build (parameters: buildParameters.ToArray ()), "`dotnet build -c Release` should succeed"); + + var runParameters = buildParameters + .Select (p => $"/p:{p}") + .Concat (["--", "--job", "Dry", "--filter", "*StringBenchmarks*"]) + .ToArray (); + using var process = dotnet.StartRun (waitForExit: true, parameters: runParameters); + + var locker = new Lock (); + var output = new StringBuilder (); + + process.OutputDataReceived += (sender, e) => { + if (e.Data != null) + lock (locker) + output.AppendLine (e.Data); + }; + process.ErrorDataReceived += (sender, e) => { + if (e.Data != null) + lock (locker) + output.AppendLine ($"STDERR: {e.Data}"); + }; + + process.BeginOutputReadLine (); + process.BeginErrorReadLine (); + + bool completed = process.WaitForExit ((int) TimeSpan.FromMinutes (15).TotalMilliseconds); + if (!completed) { + process.Kill (entireProcessTree: true); + } else { + process.WaitForExit (); + } + + string logPath = Path.Combine (projectDirectory, "dotnet-run-output.log"); + File.WriteAllText (logPath, output.ToString ()); + TestContext.AddTestAttachment (logPath); + + Assert.IsTrue (completed, $"`dotnet run -c Release` did not complete in time. See {logPath} for details."); + Assert.AreEqual (0, process.ExitCode, $"`dotnet run -c Release` should succeed. See {logPath} for details."); + + var outputText = output.ToString (); + StringAssert.Contains ("BenchmarkDotNet completed", outputText, $"Output should include the BenchmarkDotNet summary. See {logPath} for details."); + StringAssert.Contains ("BenchmarkDotNet artifacts:", outputText, $"Output should include the local artifacts path. See {logPath} for details."); + + var artifactsPath = Path.Combine (projectDirectory, "BenchmarkDotNet.Artifacts"); + Assert.IsTrue (Directory.Exists (artifactsPath), $"Expected artifacts directory '{artifactsPath}' to exist. See {logPath} for details."); + Assert.IsTrue (Directory.EnumerateFiles (artifactsPath, "*", SearchOption.AllDirectories).Any (), + $"Expected pulled BenchmarkDotNet artifacts under '{artifactsPath}'. See {logPath} for details."); + } + static int ParseInstrumentationResult (string output, string key) { // Parses lines like: INSTRUMENTATION_RESULT: passed=1 From b7d26f2b6aa8b0c0d578739274077146d7e284f1 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Sat, 9 May 2026 17:42:19 +0200 Subject: [PATCH 2/8] Generalize instrumentation run argument bridge Use a dotnet-run scoped argument bundle key and generic artifact messages in Microsoft.Android.Run. Keep BenchmarkDotNet-specific argument parsing in the Android BenchmarkDotNet template instrumentation class. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/Microsoft.Android.Run/Program.cs | 16 ++++++++-------- .../BenchmarkInstrumentation.cs | 14 +++++++------- .../Tests/InstallAndRunTests.cs | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.Android.Run/Program.cs b/src/Microsoft.Android.Run/Program.cs index 8ae2e28d475..63ca07c07cb 100644 --- a/src/Microsoft.Android.Run/Program.cs +++ b/src/Microsoft.Android.Run/Program.cs @@ -5,7 +5,7 @@ const string Name = "Microsoft.Android.Run"; const string VersionsFileName = "Microsoft.Android.versions.txt"; -const string BenchmarkArgsKey = "benchmarkArgsBase64"; +const string DotNetRunArgumentsKey = "dotnetRunArgumentsBase64"; string? adbPath = null; string? adbTarget = null; @@ -220,9 +220,9 @@ async Task RunInstrumentationAsync (List instrumentationArguments) { // Build the am instrument command var userArg = string.IsNullOrEmpty (deviceUserId) ? "" : $" --user {deviceUserId}"; - var benchmarkArgs = EncodeBenchmarkArguments (instrumentationArguments); - var benchmarkArg = benchmarkArgs == null ? "" : $" -e {BenchmarkArgsKey} {benchmarkArgs}"; - var cmdArgs = $"shell am instrument -w{userArg}{benchmarkArg} {package}/{instrumentation}"; + var encodedArguments = EncodeRunArguments (instrumentationArguments); + var argumentsArg = encodedArguments == null ? "" : $" -e {DotNetRunArgumentsKey} {encodedArguments}"; + var cmdArgs = $"shell am instrument -w{userArg}{argumentsArg} {package}/{instrumentation}"; if (verbose) Console.WriteLine ($"Running instrumentation: adb {cmdArgs}"); @@ -298,7 +298,7 @@ async Task RunInstrumentationAsync (List instrumentationArguments) if (!string.IsNullOrEmpty (result.ArtifactsPath)) { var localArtifactsPath = await PullArtifactsAsync (result.ArtifactsPath); if (!string.IsNullOrEmpty (localArtifactsPath)) - Console.WriteLine ($"BenchmarkDotNet artifacts: {localArtifactsPath}"); + Console.WriteLine ($"Artifacts: {localArtifactsPath}"); } if (!result.Succeeded) { @@ -314,7 +314,7 @@ async Task RunInstrumentationAsync (List instrumentationArguments) return 0; } -static string? EncodeBenchmarkArguments (List arguments) +static string? EncodeRunArguments (List arguments) { if (arguments.Count == 0) return null; @@ -366,7 +366,7 @@ InstrumentationRunResult ParseInstrumentationRunOutput (string output, string er var localArtifactsPath = Path.Combine (Environment.CurrentDirectory, Path.GetFileName (deviceArtifactsPath.TrimEnd ('/'))); if (verbose) - Console.WriteLine ($"Pulling BenchmarkDotNet artifacts: {deviceArtifactsPath} -> {localArtifactsPath}"); + Console.WriteLine ($"Pulling artifacts: {deviceArtifactsPath} -> {localArtifactsPath}"); var (exitCode, output, error) = await AdbHelper.RunAsync ( adbPath, @@ -375,7 +375,7 @@ InstrumentationRunResult ParseInstrumentationRunOutput (string output, string er cts.Token, verbose); if (exitCode != 0) { - Console.Error.WriteLine ($"Error: Failed to pull BenchmarkDotNet artifacts: {error}"); + Console.Error.WriteLine ($"Error: Failed to pull artifacts: {error}"); if (verbose && !string.IsNullOrWhiteSpace (output)) Console.Error.WriteLine (output); return null; diff --git a/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs b/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs index 8416e7a040d..ae45deff353 100644 --- a/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs +++ b/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs @@ -15,7 +15,7 @@ namespace AndroidBenchmark1; [Instrumentation (Name = "com.companyname.AndroidBenchmark1.BenchmarkInstrumentation")] public class BenchmarkInstrumentation : Instrumentation { - const string BenchmarkArgsKey = "benchmarkArgsBase64"; + const string DotNetRunArgumentsKey = "dotnetRunArgumentsBase64"; Bundle? arguments; @@ -42,8 +42,8 @@ public override void OnStart () var artifactsPath = Path.Combine (writablePath, "BenchmarkDotNet.Artifacts"); Directory.CreateDirectory (artifactsPath); - var benchmarkArgs = DecodeBenchmarkArguments (arguments); - var summaries = RunBenchmarks (benchmarkArgs, artifactsPath); + var runArguments = DecodeRunArguments (arguments); + var summaries = RunBenchmarks (runArguments, artifactsPath); var benchmarkCount = summaries.Sum (summary => summary.Reports.Length); var failed = summaries.Length == 0 || summaries.Any (summary => summary.HasCriticalValidationErrors || summary.Reports.Any (report => !report.Success)); @@ -58,9 +58,9 @@ public override void OnStart () }); } - static string [] DecodeBenchmarkArguments (Bundle? arguments) + static string [] DecodeRunArguments (Bundle? arguments) { - var encodedArgs = arguments?.GetString (BenchmarkArgsKey); + var encodedArgs = arguments?.GetString (DotNetRunArgumentsKey); if (string.IsNullOrEmpty (encodedArgs)) return []; @@ -68,8 +68,8 @@ static string [] DecodeBenchmarkArguments (Bundle? arguments) var decodedArgs = Encoding.UTF8.GetString (Convert.FromBase64String (encodedArgs)); return decodedArgs.Length == 0 ? [] : decodedArgs.Split ('\0'); } catch (FormatException ex) { - throw new InvalidOperationException ($"Invalid BenchmarkDotNet argument payload in '{BenchmarkArgsKey}'.", ex); - } + throw new InvalidOperationException ($"Invalid run argument payload in '{DotNetRunArgumentsKey}'.", ex); + } } static Summary [] RunBenchmarks (string [] benchmarkArgs, string artifactsPath) diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs index 8363594d372..f2d356d899f 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs @@ -2403,7 +2403,7 @@ public void DotNetNewAndroidBenchmarkDotNet () var outputText = output.ToString (); StringAssert.Contains ("BenchmarkDotNet completed", outputText, $"Output should include the BenchmarkDotNet summary. See {logPath} for details."); - StringAssert.Contains ("BenchmarkDotNet artifacts:", outputText, $"Output should include the local artifacts path. See {logPath} for details."); + StringAssert.Contains ("Artifacts:", outputText, $"Output should include the local artifacts path. See {logPath} for details."); var artifactsPath = Path.Combine (projectDirectory, "BenchmarkDotNet.Artifacts"); Assert.IsTrue (Directory.Exists (artifactsPath), $"Expected artifacts directory '{artifactsPath}' to exist. See {logPath} for details."); From 1f07537d39b6a8f389fa5c3a487985cf2c2625ce Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Sat, 9 May 2026 20:28:18 +0200 Subject: [PATCH 3/8] Document forwarded run argument encoding Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/Microsoft.Android.Run/Program.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Microsoft.Android.Run/Program.cs b/src/Microsoft.Android.Run/Program.cs index 63ca07c07cb..57575a6cbf1 100644 --- a/src/Microsoft.Android.Run/Program.cs +++ b/src/Microsoft.Android.Run/Program.cs @@ -319,6 +319,9 @@ async Task RunInstrumentationAsync (List instrumentationArguments) if (arguments.Count == 0) return null; + // These are the app arguments forwarded after `dotnet run --`. + // `am instrument -e` accepts a single string value, so preserve argv boundaries + // with NUL separators and base64-encode the payload for adb shell transport. var joinedArguments = string.Join ('\0', arguments); return Convert.ToBase64String (Encoding.UTF8.GetBytes (joinedArguments)); } From 622a8324e94afdb756ed9051023d09b725dc88ca Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Sat, 9 May 2026 22:49:42 +0200 Subject: [PATCH 4/8] Support Android BenchmarkDotNet in-process emit toolchain Allow the Android BenchmarkDotNet template to select either BenchmarkDotNet in-process toolchain with --toolchain emit|noemit. Keep noemit as the default, and document why Android instrumentation cannot use BenchmarkDotNet's external SDK-based toolchains. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../BenchmarkInstrumentation.cs | 32 +++++++++++++++++-- .../Tests/InstallAndRunTests.cs | 2 +- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs b/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs index ae45deff353..6d75a3a118b 100644 --- a/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs +++ b/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs @@ -8,6 +8,7 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.InProcess.Emit; using BenchmarkDotNet.Toolchains.InProcess.NoEmit; namespace AndroidBenchmark1; @@ -69,15 +70,22 @@ static string [] DecodeRunArguments (Bundle? arguments) return decodedArgs.Length == 0 ? [] : decodedArgs.Split ('\0'); } catch (FormatException ex) { throw new InvalidOperationException ($"Invalid run argument payload in '{DotNetRunArgumentsKey}'.", ex); - } + } } static Summary [] RunBenchmarks (string [] benchmarkArgs, string artifactsPath) { var logger = ConsoleLogger.Default; var options = ParseBenchmarkArguments (benchmarkArgs); + // BenchmarkDotNet's default toolchains build and launch a separate executable + // with the dotnet SDK. Android devices do not provide that SDK-side environment, + // so instrumentation-hosted benchmarks must run in the already-launched process. + // Use --toolchain emit|noemit to choose between BenchmarkDotNet's in-process toolchains. + var job = options.Toolchain == BenchmarkToolchain.InProcessEmit + ? options.Job.WithToolchain (InProcessEmitToolchain.Instance) + : options.Job.WithToolchain (InProcessNoEmitToolchain.Instance); var config = DefaultConfig.Instance - .AddJob (options.Job.WithToolchain (InProcessNoEmitToolchain.Instance)) + .AddJob (job) .WithArtifactsPath (artifactsPath); if (options.Filters.Count > 0) config = config.AddFilter (new GlobFilter (options.Filters.ToArray ())); @@ -103,8 +111,12 @@ static BenchmarkOptions ParseBenchmarkArguments (string [] benchmarkArgs) options.Job = GetJob (GetRequiredArgumentValue (benchmarkArgs, ref i, argument)); } else if (argument.StartsWith ("--job=", StringComparison.Ordinal)) { options.Job = GetJob (argument.Substring ("--job=".Length)); + } else if (argument == "--toolchain") { + options.Toolchain = GetToolchain (GetRequiredArgumentValue (benchmarkArgs, ref i, argument)); + } else if (argument.StartsWith ("--toolchain=", StringComparison.Ordinal)) { + options.Toolchain = GetToolchain (argument.Substring ("--toolchain=".Length)); } else { - throw new NotSupportedException ($"Unsupported BenchmarkDotNet argument '{argument}'. The Android BenchmarkDotNet template currently supports --filter/-f and --job/-j."); + throw new NotSupportedException ($"Unsupported BenchmarkDotNet argument '{argument}'. The Android BenchmarkDotNet template currently supports --filter/-f, --job/-j, and --toolchain."); } } @@ -128,9 +140,23 @@ static Job GetJob (string value) => _ => throw new ArgumentException ($"Unsupported BenchmarkDotNet job '{value}'. Supported jobs are Default, Dry, Short, Medium, and Long."), }; + static BenchmarkToolchain GetToolchain (string value) => + value.ToLowerInvariant () switch { + "emit" => BenchmarkToolchain.InProcessEmit, + "noemit" => BenchmarkToolchain.InProcessNoEmit, + _ => throw new ArgumentException ($"Unsupported BenchmarkDotNet toolchain '{value}'. Supported toolchains are Emit and NoEmit."), + }; + + enum BenchmarkToolchain + { + InProcessNoEmit, + InProcessEmit, + } + sealed class BenchmarkOptions { public Job Job { get; set; } = Job.Default; + public BenchmarkToolchain Toolchain { get; set; } public List Filters { get; } = []; } } diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs index f2d356d899f..1ecec062ddf 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs @@ -2366,7 +2366,7 @@ public void DotNetNewAndroidBenchmarkDotNet () var runParameters = buildParameters .Select (p => $"/p:{p}") - .Concat (["--", "--job", "Dry", "--filter", "*StringBenchmarks*"]) + .Concat (["--", "--job", "Dry", "--filter", "*StringBenchmarks*", "--toolchain", "noemit"]) .ToArray (); using var process = dotnet.StartRun (waitForExit: true, parameters: runParameters); From 5d930a6094ba4cd5cd6b25f62f3cae9956b40c6d Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Sun, 10 May 2026 16:38:19 +0200 Subject: [PATCH 5/8] Simplify Android BenchmarkDotNet template PR Keep the BenchmarkDotNet package version local to the template, remove the temporary template hive test helper path, and use a single in-process BenchmarkDotNet toolchain default. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/Versions.props | 1 - .../BenchmarkInstrumentation.cs | 27 +------ .../Xamarin.ProjectTools/Common/DotNetCLI.cs | 20 +---- .../MSBuildDeviceIntegration.csproj | 4 - .../Tests/InstallAndRunTests.cs | 79 ------------------- 5 files changed, 3 insertions(+), 128 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index 8568fccf4b5..2dfce5d2459 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -15,7 +15,6 @@ $(MicrosoftNETWorkloadEmscriptenCurrentManifest110100preview4PackageVersion) 11.0.100-preview.5.26251.112 0.11.5-preview.26251.112 - 0.15.8 4.3.0-preview.26252.2 10.0.7 11.0.0-preview.1.26104.118 diff --git a/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs b/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs index 6d75a3a118b..fcac00d4d5d 100644 --- a/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs +++ b/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs @@ -8,7 +8,6 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.InProcess.Emit; using BenchmarkDotNet.Toolchains.InProcess.NoEmit; namespace AndroidBenchmark1; @@ -80,12 +79,8 @@ static Summary [] RunBenchmarks (string [] benchmarkArgs, string artifactsPath) // BenchmarkDotNet's default toolchains build and launch a separate executable // with the dotnet SDK. Android devices do not provide that SDK-side environment, // so instrumentation-hosted benchmarks must run in the already-launched process. - // Use --toolchain emit|noemit to choose between BenchmarkDotNet's in-process toolchains. - var job = options.Toolchain == BenchmarkToolchain.InProcessEmit - ? options.Job.WithToolchain (InProcessEmitToolchain.Instance) - : options.Job.WithToolchain (InProcessNoEmitToolchain.Instance); var config = DefaultConfig.Instance - .AddJob (job) + .AddJob (options.Job.WithToolchain (InProcessNoEmitToolchain.Instance)) .WithArtifactsPath (artifactsPath); if (options.Filters.Count > 0) config = config.AddFilter (new GlobFilter (options.Filters.ToArray ())); @@ -111,12 +106,8 @@ static BenchmarkOptions ParseBenchmarkArguments (string [] benchmarkArgs) options.Job = GetJob (GetRequiredArgumentValue (benchmarkArgs, ref i, argument)); } else if (argument.StartsWith ("--job=", StringComparison.Ordinal)) { options.Job = GetJob (argument.Substring ("--job=".Length)); - } else if (argument == "--toolchain") { - options.Toolchain = GetToolchain (GetRequiredArgumentValue (benchmarkArgs, ref i, argument)); - } else if (argument.StartsWith ("--toolchain=", StringComparison.Ordinal)) { - options.Toolchain = GetToolchain (argument.Substring ("--toolchain=".Length)); } else { - throw new NotSupportedException ($"Unsupported BenchmarkDotNet argument '{argument}'. The Android BenchmarkDotNet template currently supports --filter/-f, --job/-j, and --toolchain."); + throw new NotSupportedException ($"Unsupported BenchmarkDotNet argument '{argument}'. The Android BenchmarkDotNet template currently supports --filter/-f and --job/-j."); } } @@ -140,23 +131,9 @@ static Job GetJob (string value) => _ => throw new ArgumentException ($"Unsupported BenchmarkDotNet job '{value}'. Supported jobs are Default, Dry, Short, Medium, and Long."), }; - static BenchmarkToolchain GetToolchain (string value) => - value.ToLowerInvariant () switch { - "emit" => BenchmarkToolchain.InProcessEmit, - "noemit" => BenchmarkToolchain.InProcessNoEmit, - _ => throw new ArgumentException ($"Unsupported BenchmarkDotNet toolchain '{value}'. Supported toolchains are Emit and NoEmit."), - }; - - enum BenchmarkToolchain - { - InProcessNoEmit, - InProcessEmit, - } - sealed class BenchmarkOptions { public Job Job { get; set; } = Job.Default; - public BenchmarkToolchain Toolchain { get; set; } public List Filters { get; } = []; } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs index 9e8fcbed405..47e5b4f90d1 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/DotNetCLI.cs @@ -105,31 +105,13 @@ protected bool Execute (params string [] args) return succeeded; } - public bool New (string template, string output = null, string customHive = null) + public bool New (string template, string output = null) { var arguments = new List { "new", template, "--output", $"\"{output ?? ProjectDirectory}\"", }; - if (!string.IsNullOrEmpty (customHive)) { - arguments.Add ("--debug:custom-hive"); - arguments.Add ($"\"{customHive}\""); - } - return Execute (arguments.ToArray ()); - } - - public bool NewInstall (string templateSource, string customHive = null) - { - var arguments = new List { - "new", - "install", - $"\"{templateSource}\"", - }; - if (!string.IsNullOrEmpty (customHive)) { - arguments.Add ("--debug:custom-hive"); - arguments.Add ($"\"{customHive}\""); - } return Execute (arguments.ToArray ()); } diff --git a/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj b/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj index baeeeedaad6..051786dd116 100644 --- a/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj +++ b/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj @@ -50,10 +50,6 @@ <_Parameter1>MSTestPackageVersion <_Parameter2>$(MSTestPackageVersion) - - <_Parameter1>BenchmarkDotNetPackageVersion - <_Parameter2>$(BenchmarkDotNetPackageVersion) - diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs index 1ecec062ddf..f178cda5a5a 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs @@ -2332,85 +2332,6 @@ public void DotNetNewAndroidTest (string mode, AndroidRuntime runtime) } } - [Test] - public void DotNetNewAndroidBenchmarkDotNet () - { - const string templateName = "DotNetNewAndroidBenchmarkDotNet"; - var projectDirectory = Path.Combine (Root, "temp", templateName); - if (Directory.Exists (projectDirectory)) - Directory.Delete (projectDirectory, true); - - TestOutputDirectories [TestContext.CurrentContext.Test.ID] = projectDirectory; - var dotnet = new DotNetCLI (Path.Combine (projectDirectory, $"{templateName}.csproj")); - var templateHive = Path.Combine (projectDirectory, ".template-hive"); - var templateSource = Path.Combine (XABuildPaths.TopDirectory, "src", "Microsoft.Android.Templates"); - Assert.IsTrue (dotnet.NewInstall (templateSource, templateHive), $"`dotnet new install {templateSource}` should succeed"); - Assert.IsTrue (dotnet.New ("android-benchmarkdotnet", customHive: templateHive), "`dotnet new android-benchmarkdotnet` should succeed"); - - var benchmarkDotNetVersion = GetAssemblyMetadataValue ("BenchmarkDotNetPackageVersion"); - var csprojPath = Path.Combine (projectDirectory, $"{templateName}.csproj"); - var doc = XDocument.Load (csprojPath); - var ns = doc.Root?.Name.Namespace ?? XNamespace.None; - var benchmarkDotNetRef = doc.Descendants (ns + "PackageReference") - .FirstOrDefault (e => e.Attribute ("Include")?.Value == "BenchmarkDotNet"); - Assert.IsNotNull (benchmarkDotNetRef, "BenchmarkDotNet PackageReference should exist in the generated project"); - benchmarkDotNetRef.SetAttributeValue ("Version", benchmarkDotNetVersion); - doc.Save (csprojPath); - - var buildParameters = new List { - "Configuration=Release", - "UseMonoRuntime=false", - "RestoreAdditionalProjectSources=https://api.nuget.org/v3/index.json", - }; - Assert.IsTrue (dotnet.Build (parameters: buildParameters.ToArray ()), "`dotnet build -c Release` should succeed"); - - var runParameters = buildParameters - .Select (p => $"/p:{p}") - .Concat (["--", "--job", "Dry", "--filter", "*StringBenchmarks*", "--toolchain", "noemit"]) - .ToArray (); - using var process = dotnet.StartRun (waitForExit: true, parameters: runParameters); - - var locker = new Lock (); - var output = new StringBuilder (); - - process.OutputDataReceived += (sender, e) => { - if (e.Data != null) - lock (locker) - output.AppendLine (e.Data); - }; - process.ErrorDataReceived += (sender, e) => { - if (e.Data != null) - lock (locker) - output.AppendLine ($"STDERR: {e.Data}"); - }; - - process.BeginOutputReadLine (); - process.BeginErrorReadLine (); - - bool completed = process.WaitForExit ((int) TimeSpan.FromMinutes (15).TotalMilliseconds); - if (!completed) { - process.Kill (entireProcessTree: true); - } else { - process.WaitForExit (); - } - - string logPath = Path.Combine (projectDirectory, "dotnet-run-output.log"); - File.WriteAllText (logPath, output.ToString ()); - TestContext.AddTestAttachment (logPath); - - Assert.IsTrue (completed, $"`dotnet run -c Release` did not complete in time. See {logPath} for details."); - Assert.AreEqual (0, process.ExitCode, $"`dotnet run -c Release` should succeed. See {logPath} for details."); - - var outputText = output.ToString (); - StringAssert.Contains ("BenchmarkDotNet completed", outputText, $"Output should include the BenchmarkDotNet summary. See {logPath} for details."); - StringAssert.Contains ("Artifacts:", outputText, $"Output should include the local artifacts path. See {logPath} for details."); - - var artifactsPath = Path.Combine (projectDirectory, "BenchmarkDotNet.Artifacts"); - Assert.IsTrue (Directory.Exists (artifactsPath), $"Expected artifacts directory '{artifactsPath}' to exist. See {logPath} for details."); - Assert.IsTrue (Directory.EnumerateFiles (artifactsPath, "*", SearchOption.AllDirectories).Any (), - $"Expected pulled BenchmarkDotNet artifacts under '{artifactsPath}'. See {logPath} for details."); - } - static int ParseInstrumentationResult (string output, string key) { // Parses lines like: INSTRUMENTATION_RESULT: passed=1 From 4f8b09d7ca26be8fe51260c6f2dc7b5fc51a503e Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Sun, 10 May 2026 22:34:27 +0200 Subject: [PATCH 6/8] Address Android BenchmarkDotNet review feedback Drain instrumentation output before parsing results, fail runs when reported artifacts cannot be pulled, validate device artifact paths before adb pull, and add focused coverage for instrumentation argument forwarding. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/Microsoft.Android.Run/Program.cs | 50 +++++++++++++++++-- .../Xamarin.Android.Build.Tests/XASdkTests.cs | 14 ++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Android.Run/Program.cs b/src/Microsoft.Android.Run/Program.cs index 57575a6cbf1..a6735342bd6 100644 --- a/src/Microsoft.Android.Run/Program.cs +++ b/src/Microsoft.Android.Run/Program.cs @@ -266,8 +266,14 @@ async Task RunInstrumentationAsync (List instrumentationArguments) try { try { await instrumentProcess.WaitForExitAsync (cts.Token); + instrumentProcess.WaitForExit (); } catch (OperationCanceledException) { - try { instrumentProcess.Kill (); } catch (Exception ex) { + try { + if (!instrumentProcess.HasExited) { + instrumentProcess.Kill (); + instrumentProcess.WaitForExit (1000); + } + } catch (Exception ex) { if (verbose) Console.Error.WriteLine ($"Cleanup: {ex.Message}"); } @@ -297,8 +303,9 @@ async Task RunInstrumentationAsync (List instrumentationArguments) Console.WriteLine (result.Summary); if (!string.IsNullOrEmpty (result.ArtifactsPath)) { var localArtifactsPath = await PullArtifactsAsync (result.ArtifactsPath); - if (!string.IsNullOrEmpty (localArtifactsPath)) - Console.WriteLine ($"Artifacts: {localArtifactsPath}"); + if (string.IsNullOrEmpty (localArtifactsPath)) + return 1; + Console.WriteLine ($"Artifacts: {localArtifactsPath}"); } if (!result.Succeeded) { @@ -366,7 +373,12 @@ InstrumentationRunResult ParseInstrumentationRunOutput (string output, string er async Task PullArtifactsAsync (string deviceArtifactsPath) { - var localArtifactsPath = Path.Combine (Environment.CurrentDirectory, Path.GetFileName (deviceArtifactsPath.TrimEnd ('/'))); + if (!TryGetArtifactsDirectoryName (deviceArtifactsPath, out var artifactsDirectoryName)) { + Console.Error.WriteLine ($"Error: Invalid artifacts path reported by instrumentation: '{deviceArtifactsPath}'."); + return null; + } + + var localArtifactsPath = Path.Combine (Environment.CurrentDirectory, artifactsDirectoryName); if (verbose) Console.WriteLine ($"Pulling artifacts: {deviceArtifactsPath} -> {localArtifactsPath}"); @@ -387,6 +399,36 @@ InstrumentationRunResult ParseInstrumentationRunOutput (string output, string er return localArtifactsPath; } +static bool TryGetArtifactsDirectoryName (string deviceArtifactsPath, out string artifactsDirectoryName) +{ + artifactsDirectoryName = ""; + if (string.IsNullOrEmpty (deviceArtifactsPath) || deviceArtifactsPath [0] != '/') + return false; + + foreach (var ch in deviceArtifactsPath) { + if (char.IsControl (ch) || char.IsWhiteSpace (ch)) { + return false; + } + switch (ch) { + case '"': + case '\'': + case ';': + case '&': + case '|': + case '`': + case '$': + case '<': + case '>': + case '\\': + return false; + } + } + + var trimmedPath = deviceArtifactsPath.TrimEnd ('/'); + artifactsDirectoryName = Path.GetFileName (trimmedPath); + return artifactsDirectoryName != "" && artifactsDirectoryName != "." && artifactsDirectoryName != ".."; +} + async Task RunDotnetTestAsync (List mtpArgs) { if (verbose) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs index c43bc312316..a773152f817 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs @@ -46,6 +46,20 @@ public void DotNetNew ([Values ("android", "androidlib", "android-bindinglib", " dotnet.AssertHasNoWarnings (); } + [Test] + public void ComputeRunArgumentsForwardsInstrumentationStartArguments () + { + var targetsPath = Path.Combine (XABuildPaths.TopDirectory, "src", "Xamarin.Android.Build.Tasks", "Microsoft.Android.Sdk", "targets", "Microsoft.Android.Sdk.Application.targets"); + var targets = XDocument.Load (targetsPath); + var runStartArguments = targets.Descendants ("_AndroidRunStartArguments").Single (); + var runArguments = targets.Descendants ("RunArguments") + .Single (e => e.Value.Contains ("$(_AndroidRunPath)", StringComparison.Ordinal)); + + Assert.AreEqual (" '$(AndroidInstrumentation)' != '' and '$(StartArguments)' != '' ", runStartArguments.Attribute ("Condition")?.Value); + StringAssert.Contains ("$(_AndroidRunStartArguments)", runArguments.Value); + StringAssert.Contains ("$(_AndroidRunInstrumentArg)", runArguments.Value); + } + static IEnumerable Get_DotNetPack_Data () { var ret = new List (); From b127891da602b12e35bf394f809fd090384e3d96 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Sun, 10 May 2026 22:45:47 +0200 Subject: [PATCH 7/8] Clarify forwarded run argument parsing Document that the Mono.Options <> handler captures forwarded app arguments after dotnet run --, not the delimiter itself. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/Microsoft.Android.Run/Program.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Microsoft.Android.Run/Program.cs b/src/Microsoft.Android.Run/Program.cs index a6735342bd6..1e2f140f043 100644 --- a/src/Microsoft.Android.Run/Program.cs +++ b/src/Microsoft.Android.Run/Program.cs @@ -80,6 +80,9 @@ async Task RunAsync (string[] args) { "h|help|?", "Show this help message and exit.", v => showHelp = v != null }, + // Mono.Options uses "<>" as the default handler for non-option arguments. + // `dotnet run --` consumes the `--` delimiter before this tool is launched, + // so this captures only the forwarded app arguments that follow it. { "<>", v => remaining.Add (v) }, }; From 4d4c1030acea3fec72be740e474c1b33d0f57e02 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Mon, 11 May 2026 17:27:29 +0200 Subject: [PATCH 8/8] Address review feedback from Jon - AndroidBenchmark1.csproj: replace blanket PublishTrimmed=false/PublishReadyToRun=false with TrimmerRootAssembly entries that keep BenchmarkDotNet and the user's own assembly intact while letting the rest of the app trim normally. - Microsoft.Android.Sdk.Application.targets: inline $(StartArguments) directly in RunArguments instead of introducing a dedicated _AndroidRunStartArguments property. Microsoft.Android.Run only accepts positional arguments in --instrument mode and will surface a usage error otherwise. - XASdkTests: add android-benchmarkdotnet to DotNetNew's [Values] so the new template is exercised by the build-every-template fixture, and update ComputeRunArgumentsForwardsInstrumentationStartArguments to match the simplified targets. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../android-benchmarkdotnet/AndroidBenchmark1.csproj | 11 +++++++++-- .../targets/Microsoft.Android.Sdk.Application.targets | 7 +++++-- .../Tests/Xamarin.Android.Build.Tests/XASdkTests.cs | 6 ++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidBenchmark1.csproj b/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidBenchmark1.csproj index 758dc720396..ff65fe9fc96 100644 --- a/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidBenchmark1.csproj +++ b/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidBenchmark1.csproj @@ -14,11 +14,18 @@ android-arm64;android-x64 apk com.companyname.AndroidBenchmark1.BenchmarkInstrumentation - false - false + + + + + + diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Application.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Application.targets index 9bcd92289c8..3d895eb3d33 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Application.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Application.targets @@ -127,9 +127,12 @@ This file contains targets specific for Android application projects. <_AndroidRunUserArg Condition=" '$(AndroidDeviceUserId)' != '' ">--user "$(AndroidDeviceUserId)" <_AndroidRunInstrumentArg Condition=" '$(AndroidInstrumentation)' != '' ">--instrument "$(AndroidInstrumentation)" <_AndroidRunActivityArg Condition=" '$(AndroidInstrumentation)' == '' ">--activity "$(AndroidLaunchActivity)" - <_AndroidRunStartArguments Condition=" '$(AndroidInstrumentation)' != '' and '$(StartArguments)' != '' ">$(StartArguments) dotnet - exec "$(_AndroidRunPath)" --adb "$(_AdbToolPath)" $(_AndroidRunAdbTargetArg) --package "$(_AndroidPackage)" $(_AndroidRunActivityArg) $(_AndroidRunInstrumentArg) --logcat-args "$(_AndroidRunLogcatArgs)" $(_AndroidRunUserArg) $(_AndroidRunExtraArgs) $(_AndroidRunStartArguments) + + exec "$(_AndroidRunPath)" --adb "$(_AdbToolPath)" $(_AndroidRunAdbTargetArg) --package "$(_AndroidPackage)" $(_AndroidRunActivityArg) $(_AndroidRunInstrumentArg) --logcat-args "$(_AndroidRunLogcatArgs)" $(_AndroidRunUserArg) $(_AndroidRunExtraArgs) $(StartArguments) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs index a773152f817..8e2040e5f09 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs @@ -23,7 +23,7 @@ namespace Xamarin.Android.Build.Tests public class XASdkTests : BaseTest { [Test] - public void DotNetNew ([Values ("android", "androidlib", "android-bindinglib", "androidwear")] string template) + public void DotNetNew ([Values ("android", "androidlib", "android-bindinglib", "androidwear", "android-benchmarkdotnet")] string template) { var templateName = TestName.Replace ("-", ""); var templatePath = Path.Combine (Root, "temp", templateName); @@ -51,12 +51,10 @@ public void ComputeRunArgumentsForwardsInstrumentationStartArguments () { var targetsPath = Path.Combine (XABuildPaths.TopDirectory, "src", "Xamarin.Android.Build.Tasks", "Microsoft.Android.Sdk", "targets", "Microsoft.Android.Sdk.Application.targets"); var targets = XDocument.Load (targetsPath); - var runStartArguments = targets.Descendants ("_AndroidRunStartArguments").Single (); var runArguments = targets.Descendants ("RunArguments") .Single (e => e.Value.Contains ("$(_AndroidRunPath)", StringComparison.Ordinal)); - Assert.AreEqual (" '$(AndroidInstrumentation)' != '' and '$(StartArguments)' != '' ", runStartArguments.Attribute ("Condition")?.Value); - StringAssert.Contains ("$(_AndroidRunStartArguments)", runArguments.Value); + StringAssert.Contains ("$(StartArguments)", runArguments.Value); StringAssert.Contains ("$(_AndroidRunInstrumentArg)", runArguments.Value); }