diff --git a/src/Microsoft.Android.Run/Program.cs b/src/Microsoft.Android.Run/Program.cs index 2da984571d2..22ef5a9e4a8 100644 --- a/src/Microsoft.Android.Run/Program.cs +++ b/src/Microsoft.Android.Run/Program.cs @@ -1,9 +1,11 @@ using System.Diagnostics; +using System.Text; using Mono.Options; using Xamarin.Android.Tools; const string Name = "Microsoft.Android.Run"; const string VersionsFileName = "Microsoft.Android.versions.txt"; +const string DotNetRunArgumentsKey = "dotnetRunArgumentsBase64"; string? adbPath = null; string? adbTarget = null; @@ -34,6 +36,7 @@ async Task RunAsync (string[] args) { bool showHelp = false; bool showVersion = false; + var remaining = new List (); var options = new OptionSet { $"Usage: {Name} [OPTIONS]", @@ -79,18 +82,24 @@ async Task RunAsync (string[] args) { "h|help|?", "Show this help message and exit.", v => showHelp = v != null }, + // Mono.Options uses "<>" as the default handler for non-option arguments. + // `dotnet run --` consumes the `--` delimiter before this tool is launched, + // so this captures only the forwarded app arguments that follow it. + { "<>", + v => remaining.Add (v) }, }; - List remaining; try { - remaining = options.Parse (args); + options.Parse (args); } catch (OptionException e) { Console.Error.WriteLine ($"Error: {e.Message}"); Console.Error.WriteLine ($"Try '{Name} --help' for more information."); return 1; } - if (remaining.Count > 0 && !isDotnetTestMode) { + bool isInstrumentMode = !string.IsNullOrEmpty (instrumentation); + + if (remaining.Count > 0 && !isDotnetTestMode && !isInstrumentMode) { Console.Error.WriteLine ($"Error: Unexpected argument(s): {string.Join (" ", remaining)}"); Console.Error.WriteLine ($"Try '{Name} --help' for more information."); return 1; @@ -126,8 +135,6 @@ async Task RunAsync (string[] args) return 1; } - bool isInstrumentMode = !string.IsNullOrEmpty (instrumentation); - if (!isInstrumentMode && string.IsNullOrEmpty (activity) && !isDotnetTestMode) { Console.Error.WriteLine ("Error: --activity or --instrument is required."); Console.Error.WriteLine ($"Try '{Name} --help' for more information."); @@ -183,7 +190,7 @@ async Task RunAsync (string[] args) return await RunDotnetTestAsync (remaining); if (isInstrumentMode) - return await RunInstrumentationAsync (); + return await RunInstrumentationAsync (remaining); return await RunAppAsync (); } finally { @@ -214,11 +221,13 @@ void OnCancelKeyPress (object? sender, ConsoleCancelEventArgs e) } } -async Task RunInstrumentationAsync () +async Task RunInstrumentationAsync (List instrumentationArguments) { // Build the am instrument command var userArg = string.IsNullOrEmpty (deviceUserId) ? "" : $" --user {deviceUserId}"; - var cmdArgs = $"shell am instrument -w{userArg} {package}/{instrumentation}"; + var encodedArguments = EncodeRunArguments (instrumentationArguments); + var argumentsArg = encodedArguments == null ? "" : $" -e {DotNetRunArgumentsKey} {encodedArguments}"; + var cmdArgs = $"shell am instrument -w{userArg}{argumentsArg} {package}/{instrumentation}"; if (verbose) Console.WriteLine ($"Running instrumentation: adb {cmdArgs}"); @@ -228,17 +237,25 @@ async Task RunInstrumentationAsync () using var instrumentProcess = new Process { StartInfo = psi }; var locker = new Lock (); + var output = new StringBuilder (); + var error = new StringBuilder (); instrumentProcess.OutputDataReceived += (s, e) => { - if (e.Data != null) - lock (locker) + if (e.Data != null) { + lock (locker) { + output.AppendLine (e.Data); Console.WriteLine (e.Data); + } + } }; instrumentProcess.ErrorDataReceived += (s, e) => { - if (e.Data != null) - lock (locker) + if (e.Data != null) { + lock (locker) { + error.AppendLine (e.Data); Console.Error.WriteLine (e.Data); + } + } }; instrumentProcess.Start (); @@ -254,8 +271,14 @@ async Task RunInstrumentationAsync () try { try { await instrumentProcess.WaitForExitAsync (cts.Token); + instrumentProcess.WaitForExit (); } catch (OperationCanceledException) { - try { instrumentProcess.Kill (); } catch (Exception ex) { + try { + if (!instrumentProcess.HasExited) { + instrumentProcess.Kill (); + instrumentProcess.WaitForExit (1000); + } + } catch (Exception ex) { if (verbose) Console.Error.WriteLine ($"Cleanup: {ex.Message}"); } @@ -280,9 +303,137 @@ async Task RunInstrumentationAsync () return 1; } + var result = ParseInstrumentationRunOutput (output.ToString (), error.ToString ()); + if (!string.IsNullOrEmpty (result.Summary)) + Console.WriteLine (result.Summary); + if (!string.IsNullOrEmpty (result.ArtifactsPath)) { + var localArtifactsPath = await PullArtifactsAsync (result.ArtifactsPath); + if (string.IsNullOrEmpty (localArtifactsPath)) + return 1; + Console.WriteLine ($"Artifacts: {localArtifactsPath}"); + } + + if (!result.Succeeded) { + if (!string.IsNullOrEmpty (result.Error)) + Console.Error.WriteLine (result.Error); + else if (result.InstrumentationCode.HasValue) + Console.Error.WriteLine ($"Instrumentation failed with code {result.InstrumentationCode.Value}."); + else + Console.Error.WriteLine ("Instrumentation failed."); + return 1; + } + return 0; } +static string? EncodeRunArguments (List arguments) +{ + if (arguments.Count == 0) + return null; + + // These are the app arguments forwarded after `dotnet run --`. + // `am instrument -e` accepts a single string value, so preserve argv boundaries + // with NUL separators and base64-encode the payload for adb shell transport. + var joinedArguments = string.Join ('\0', arguments); + return Convert.ToBase64String (Encoding.UTF8.GetBytes (joinedArguments)); +} + +InstrumentationRunResult ParseInstrumentationRunOutput (string output, string error) +{ + var result = new InstrumentationRunResult (); + foreach (var rawLine in (output + "\n" + error).Split ('\n')) { + var line = rawLine.TrimEnd ('\r'); + if (line.StartsWith ("INSTRUMENTATION_RESULT: ", StringComparison.Ordinal)) { + var value = line.Substring ("INSTRUMENTATION_RESULT: ".Length); + var separator = value.IndexOf ('='); + if (separator <= 0) + continue; + + var key = value.Substring (0, separator).Trim (); + var keyValue = value.Substring (separator + 1).Trim (); + switch (key) { + case "artifactsPath": + result.ArtifactsPath = keyValue; + break; + case "summary": + result.Summary = keyValue; + break; + case "error": + result.Error = keyValue; + break; + } + } else if (line.StartsWith ("INSTRUMENTATION_CODE: ", StringComparison.Ordinal)) { + var code = line.Substring ("INSTRUMENTATION_CODE: ".Length).Trim (); + if (int.TryParse (code, out int instrumentationCode)) + result.InstrumentationCode = instrumentationCode; + } else if (line.StartsWith ("INSTRUMENTATION_FAILED", StringComparison.Ordinal)) { + result.InstrumentationFailed = true; + if (string.IsNullOrEmpty (result.Error)) + result.Error = line; + } + } + + return result; +} + +async Task PullArtifactsAsync (string deviceArtifactsPath) +{ + if (!TryGetArtifactsDirectoryName (deviceArtifactsPath, out var artifactsDirectoryName)) { + Console.Error.WriteLine ($"Error: Invalid artifacts path reported by instrumentation: '{deviceArtifactsPath}'."); + return null; + } + + var localArtifactsPath = Path.Combine (Environment.CurrentDirectory, artifactsDirectoryName); + + if (verbose) + Console.WriteLine ($"Pulling artifacts: {deviceArtifactsPath} -> {localArtifactsPath}"); + + var (exitCode, output, error) = await AdbHelper.RunAsync ( + adbPath, + adbTarget, + $"pull \"{deviceArtifactsPath}\" \"{Environment.CurrentDirectory}\"", + cts.Token, + verbose); + if (exitCode != 0) { + Console.Error.WriteLine ($"Error: Failed to pull artifacts: {error}"); + if (verbose && !string.IsNullOrWhiteSpace (output)) + Console.Error.WriteLine (output); + return null; + } + + return localArtifactsPath; +} + +static bool TryGetArtifactsDirectoryName (string deviceArtifactsPath, out string artifactsDirectoryName) +{ + artifactsDirectoryName = ""; + if (string.IsNullOrEmpty (deviceArtifactsPath) || deviceArtifactsPath [0] != '/') + return false; + + foreach (var ch in deviceArtifactsPath) { + if (char.IsControl (ch) || char.IsWhiteSpace (ch)) { + return false; + } + switch (ch) { + case '"': + case '\'': + case ';': + case '&': + case '|': + case '`': + case '$': + case '<': + case '>': + case '\\': + return false; + } + } + + var trimmedPath = deviceArtifactsPath.TrimEnd ('/'); + artifactsDirectoryName = Path.GetFileName (trimmedPath); + return artifactsDirectoryName != "" && artifactsDirectoryName != "." && artifactsDirectoryName != ".."; +} + async Task RunDotnetTestAsync (List mtpArgs) { if (verbose) @@ -522,3 +673,17 @@ async Task StopAppAsync () return (null, null); } } + +class InstrumentationRunResult +{ + public string? ArtifactsPath { get; set; } + public string? Summary { get; set; } + public string? Error { get; set; } + public int? InstrumentationCode { get; set; } + public bool InstrumentationFailed { get; set; } + + public bool Succeeded => + !InstrumentationFailed && + string.IsNullOrEmpty (Error) && + (!InstrumentationCode.HasValue || InstrumentationCode.Value == -1); +} diff --git a/src/Microsoft.Android.Templates/android-benchmarkdotnet/.template.config/template.json b/src/Microsoft.Android.Templates/android-benchmarkdotnet/.template.config/template.json new file mode 100644 index 00000000000..8d121d4300e --- /dev/null +++ b/src/Microsoft.Android.Templates/android-benchmarkdotnet/.template.config/template.json @@ -0,0 +1,34 @@ +{ + "$schema": "http://json.schemastore.org/template", + "author": "Microsoft", + "classifications": [ "Android", "Mobile", "Benchmark" ], + "identity": "Microsoft.Android.AndroidBenchmarkDotNet", + "name": "Android BenchmarkDotNet Project", + "description": "A project for creating a .NET for Android BenchmarkDotNet project", + "shortName": "android-benchmarkdotnet", + "tags": { + "language": "C#", + "type": "project" + }, + "sourceName": "AndroidBenchmark1", + "preferNameDirectory": true, + "primaryOutputs": [ + { "path": "AndroidBenchmark1.csproj" } + ], + "symbols": { + "packageName": { + "type": "parameter", + "description": "Overrides the package name in the AndroidManifest.xml", + "datatype": "string", + "replaces": "com.companyname.AndroidBenchmark1" + }, + "supportedOSVersion": { + "type": "parameter", + "description": "Overrides $(SupportedOSPlatformVersion) in the project", + "datatype": "string", + "replaces": "SUPPORTED_OS_PLATFORM_VERSION", + "defaultValue": "24" + } + }, + "defaultName": "AndroidBenchmark1" +} diff --git a/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidBenchmark1.csproj b/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidBenchmark1.csproj new file mode 100644 index 00000000000..ff65fe9fc96 --- /dev/null +++ b/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidBenchmark1.csproj @@ -0,0 +1,31 @@ + + + net11.0-android + SUPPORTED_OS_PLATFORM_VERSION + AndroidBenchmark1 + Exe + enable + none + enable + com.companyname.AndroidBenchmark1 + 1 + 1.0 + false + android-arm64;android-x64 + apk + com.companyname.AndroidBenchmark1.BenchmarkInstrumentation + + + + + + + + + + + + diff --git a/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidManifest.xml b/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidManifest.xml new file mode 100644 index 00000000000..7c071193952 --- /dev/null +++ b/src/Microsoft.Android.Templates/android-benchmarkdotnet/AndroidManifest.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs b/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs new file mode 100644 index 00000000000..fcac00d4d5d --- /dev/null +++ b/src/Microsoft.Android.Templates/android-benchmarkdotnet/BenchmarkInstrumentation.cs @@ -0,0 +1,139 @@ +using System.Text; +using Android.App; +using Android.OS; +using Android.Runtime; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Filters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.InProcess.NoEmit; + +namespace AndroidBenchmark1; + +[Instrumentation (Name = "com.companyname.AndroidBenchmark1.BenchmarkInstrumentation")] +public class BenchmarkInstrumentation : Instrumentation +{ + const string DotNetRunArgumentsKey = "dotnetRunArgumentsBase64"; + + Bundle? arguments; + + protected BenchmarkInstrumentation (IntPtr handle, JniHandleOwnership ownership) + : base (handle, ownership) + { + } + + public override void OnCreate (Bundle? arguments) + { + base.OnCreate (arguments); + this.arguments = arguments; + Start (); + } + + public override void OnStart () + { + base.OnStart (); + + Task.Run (() => { + var bundle = new Bundle (); + try { + var writablePath = Application.Context.GetExternalFilesDir (null)?.AbsolutePath ?? Path.GetTempPath (); + var artifactsPath = Path.Combine (writablePath, "BenchmarkDotNet.Artifacts"); + Directory.CreateDirectory (artifactsPath); + + var runArguments = DecodeRunArguments (arguments); + var summaries = RunBenchmarks (runArguments, artifactsPath); + var benchmarkCount = summaries.Sum (summary => summary.Reports.Length); + var failed = summaries.Length == 0 || summaries.Any (summary => summary.HasCriticalValidationErrors || summary.Reports.Any (report => !report.Success)); + + bundle.PutInt ("benchmarks", benchmarkCount); + bundle.PutString ("artifactsPath", artifactsPath); + bundle.PutString ("summary", $"BenchmarkDotNet completed {benchmarkCount} benchmark report(s)."); + Finish (failed ? Result.Canceled : Result.Ok, bundle); + } catch (Exception ex) { + bundle.PutString ("error", ex.ToString ()); + Finish (Result.Canceled, bundle); + } + }); + } + + static string [] DecodeRunArguments (Bundle? arguments) + { + var encodedArgs = arguments?.GetString (DotNetRunArgumentsKey); + if (string.IsNullOrEmpty (encodedArgs)) + return []; + + try { + var decodedArgs = Encoding.UTF8.GetString (Convert.FromBase64String (encodedArgs)); + return decodedArgs.Length == 0 ? [] : decodedArgs.Split ('\0'); + } catch (FormatException ex) { + throw new InvalidOperationException ($"Invalid run argument payload in '{DotNetRunArgumentsKey}'.", ex); + } + } + + static Summary [] RunBenchmarks (string [] benchmarkArgs, string artifactsPath) + { + var logger = ConsoleLogger.Default; + var options = ParseBenchmarkArguments (benchmarkArgs); + // BenchmarkDotNet's default toolchains build and launch a separate executable + // with the dotnet SDK. Android devices do not provide that SDK-side environment, + // so instrumentation-hosted benchmarks must run in the already-launched process. + var config = DefaultConfig.Instance + .AddJob (options.Job.WithToolchain (InProcessNoEmitToolchain.Instance)) + .WithArtifactsPath (artifactsPath); + if (options.Filters.Count > 0) + config = config.AddFilter (new GlobFilter (options.Filters.ToArray ())); + + var (allTypesValid, runnableTypes) = TypeFilter.GetTypesWithRunnableBenchmarks ([], [typeof (BenchmarkInstrumentation).Assembly], logger); + if (!allTypesValid || runnableTypes.Count == 0) + return []; + + var benchmarkRunInfos = TypeFilter.Filter (config, runnableTypes); + return BenchmarkRunner.Run (benchmarkRunInfos); + } + + static BenchmarkOptions ParseBenchmarkArguments (string [] benchmarkArgs) + { + var options = new BenchmarkOptions (); + for (int i = 0; i < benchmarkArgs.Length; i++) { + var argument = benchmarkArgs [i]; + if (argument == "-f" || argument == "--filter") { + options.Filters.Add (GetRequiredArgumentValue (benchmarkArgs, ref i, argument)); + } else if (argument.StartsWith ("--filter=", StringComparison.Ordinal)) { + options.Filters.Add (argument.Substring ("--filter=".Length)); + } else if (argument == "-j" || argument == "--job") { + options.Job = GetJob (GetRequiredArgumentValue (benchmarkArgs, ref i, argument)); + } else if (argument.StartsWith ("--job=", StringComparison.Ordinal)) { + options.Job = GetJob (argument.Substring ("--job=".Length)); + } else { + throw new NotSupportedException ($"Unsupported BenchmarkDotNet argument '{argument}'. The Android BenchmarkDotNet template currently supports --filter/-f and --job/-j."); + } + } + + return options; + } + + static string GetRequiredArgumentValue (string [] benchmarkArgs, ref int index, string argument) + { + if (index + 1 >= benchmarkArgs.Length) + throw new ArgumentException ($"Missing value for '{argument}'."); + return benchmarkArgs [++index]; + } + + static Job GetJob (string value) => + value.ToLowerInvariant () switch { + "default" => Job.Default, + "dry" => Job.Dry, + "short" => Job.ShortRun, + "medium" => Job.MediumRun, + "long" => Job.LongRun, + _ => throw new ArgumentException ($"Unsupported BenchmarkDotNet job '{value}'. Supported jobs are Default, Dry, Short, Medium, and Long."), + }; + + sealed class BenchmarkOptions + { + public Job Job { get; set; } = Job.Default; + public List Filters { get; } = []; + } +} diff --git a/src/Microsoft.Android.Templates/android-benchmarkdotnet/StringBenchmarks.cs b/src/Microsoft.Android.Templates/android-benchmarkdotnet/StringBenchmarks.cs new file mode 100644 index 00000000000..93a4a17ca53 --- /dev/null +++ b/src/Microsoft.Android.Templates/android-benchmarkdotnet/StringBenchmarks.cs @@ -0,0 +1,18 @@ +using BenchmarkDotNet.Attributes; + +namespace AndroidBenchmark1; + +[MemoryDiagnoser] +public class StringBenchmarks +{ + readonly string [] values = Enumerable.Range (0, 100).Select (i => i.ToString ()).ToArray (); + + [Benchmark] + public int ParseIntegers () + { + var sum = 0; + foreach (var value in values) + sum += int.Parse (value); + return sum; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Application.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Application.targets index 03d9295a4f8..3d895eb3d33 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Application.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.Application.targets @@ -128,7 +128,11 @@ This file contains targets specific for Android application projects. <_AndroidRunInstrumentArg Condition=" '$(AndroidInstrumentation)' != '' ">--instrument "$(AndroidInstrumentation)" <_AndroidRunActivityArg Condition=" '$(AndroidInstrumentation)' == '' ">--activity "$(AndroidLaunchActivity)" dotnet - exec "$(_AndroidRunPath)" --adb "$(_AdbToolPath)" $(_AndroidRunAdbTargetArg) --package "$(_AndroidPackage)" $(_AndroidRunActivityArg) $(_AndroidRunInstrumentArg) --logcat-args "$(_AndroidRunLogcatArgs)" $(_AndroidRunUserArg) $(_AndroidRunExtraArgs) + + exec "$(_AndroidRunPath)" --adb "$(_AdbToolPath)" $(_AndroidRunAdbTargetArg) --package "$(_AndroidPackage)" $(_AndroidRunActivityArg) $(_AndroidRunInstrumentArg) --logcat-args "$(_AndroidRunLogcatArgs)" $(_AndroidRunUserArg) $(_AndroidRunExtraArgs) $(StartArguments) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs index c43bc312316..8e2040e5f09 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/XASdkTests.cs @@ -23,7 +23,7 @@ namespace Xamarin.Android.Build.Tests public class XASdkTests : BaseTest { [Test] - public void DotNetNew ([Values ("android", "androidlib", "android-bindinglib", "androidwear")] string template) + public void DotNetNew ([Values ("android", "androidlib", "android-bindinglib", "androidwear", "android-benchmarkdotnet")] string template) { var templateName = TestName.Replace ("-", ""); var templatePath = Path.Combine (Root, "temp", templateName); @@ -46,6 +46,18 @@ public void DotNetNew ([Values ("android", "androidlib", "android-bindinglib", " dotnet.AssertHasNoWarnings (); } + [Test] + public void ComputeRunArgumentsForwardsInstrumentationStartArguments () + { + var targetsPath = Path.Combine (XABuildPaths.TopDirectory, "src", "Xamarin.Android.Build.Tasks", "Microsoft.Android.Sdk", "targets", "Microsoft.Android.Sdk.Application.targets"); + var targets = XDocument.Load (targetsPath); + var runArguments = targets.Descendants ("RunArguments") + .Single (e => e.Value.Contains ("$(_AndroidRunPath)", StringComparison.Ordinal)); + + StringAssert.Contains ("$(StartArguments)", runArguments.Value); + StringAssert.Contains ("$(_AndroidRunInstrumentArg)", runArguments.Value); + } + static IEnumerable Get_DotNetPack_Data () { var ret = new List ();