From 635915ef07e8a0fd83b8133a03cdb238f5761b1b Mon Sep 17 00:00:00 2001 From: saibulusu Date: Fri, 16 Jan 2026 16:47:02 -0800 Subject: [PATCH 1/3] speccpu limitations on windows, specifying benchmarks for SpeccpuExecutor --- .../SPECcpu/SpecCpuExecutor.cs | 14 +++++++++++++- .../ProfileExpressionEvaluatorTests.cs | 16 ++++++++++++++++ .../profiles/PERF-SPECCPU-FPRATE.json | 15 +++++++++------ .../profiles/PERF-SPECCPU-FPSPEED.json | 15 +++++++++------ .../profiles/PERF-SPECCPU-INTRATE.json | 15 +++++++++------ .../profiles/PERF-SPECCPU-INTSPEED.json | 15 +++++++++------ .../docs/workloads/speccpu/speccpu-profiles.md | 12 ++++++++++++ 7 files changed, 77 insertions(+), 25 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuExecutor.cs b/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuExecutor.cs index bea923efa3..8c9c8562d5 100644 --- a/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuExecutor.cs @@ -64,6 +64,18 @@ public string SpecProfile } } + /// + /// List of benchmarks to run. + /// + public string Benchmarks + { + get + { + this.Parameters.TryGetValue(nameof(SpecCpuExecutor.Benchmarks), out IConvertible benchmarks); + return benchmarks?.ToString(); + } + } + /// /// The whether SPECcpu runs base tuning or base+peak tuning. /// @@ -396,7 +408,7 @@ private string GetCommandLineArguments() // Iterations has to be either 2 or 3 for reportable runs. https://www.spec.org/cpu2017/Docs/config.html#reportable bool reportable = (this.Platform == PlatformID.Unix) && (this.Iterations == 2 || this.Iterations == 3); cmd = reportable ? $"{cmd} --reportable" : $"{cmd} --noreportable"; - cmd = $"{cmd} {this.SpecProfile}"; + cmd = $"{cmd} {this.Benchmarks}"; return cmd; } diff --git a/src/VirtualClient/VirtualClient.Core.UnitTests/ProfileExpressionEvaluatorTests.cs b/src/VirtualClient/VirtualClient.Core.UnitTests/ProfileExpressionEvaluatorTests.cs index abe91ed868..64e8dd6806 100644 --- a/src/VirtualClient/VirtualClient.Core.UnitTests/ProfileExpressionEvaluatorTests.cs +++ b/src/VirtualClient/VirtualClient.Core.UnitTests/ProfileExpressionEvaluatorTests.cs @@ -1440,6 +1440,22 @@ public async Task ProfileExpressionEvaluatorSupportsTernaryFunctionReferencesInP Assert.AreEqual(true, parameters["IsTLSEnabled"]); } + [Test] + public async Task ProfileExpressionEvaluatorSupportsTernaryFunctionReferencesInParameterSets_Scenario_7() + { + this.SetupDefaults(PlatformID.Win32NT); + + Dictionary parameters = new Dictionary + { + { "Benchmarks" , "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"intrate\" : \"505 525 541 548 557\")}" }, + }; + + await ProfileExpressionEvaluator.Instance.EvaluateAsync(this.mockFixture.Dependencies, parameters); + + Assert.AreEqual("505 525 541 548 557", parameters["Benchmarks"]); + } + + [Test] public async Task ProfileExpressionEvaluatorSupportsNestedTernaryFunctionReferencesInParameterSets_Scenario_1() diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json index 4175096521..3afc69b0fb 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json @@ -6,12 +6,12 @@ "RecommendedMinimumExecutionTime": "(4-cores)=02:00:00,(16-cores)=05:00:00,(64-cores)=10:00:00", "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows", - "Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24" + "Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24", + "Limitations": "The following benchmarks don't work on Windows: 507, 511, 521, 526, 527, 538" }, "Parameters": { "CompilerVersion": "", "Iterations": 2, - "SpecProfile": "fprate", "RunPeak": false, "Threads": "{LogicalCoreCount}", "Copies": "{LogicalCoreCount}", @@ -24,7 +24,8 @@ "Parameters": { "Scenario": "ExecuteSPECBenchmark", "Iterations": "$.Parameters.Iterations", - "SpecProfile": "$.Parameters.SpecProfile", + "SpecProfile": "fprate", + "Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"fprate\" : \"503 508 510 519 544 549 554\")}", "PackageName": "speccpu2017", "RunPeak": "$.Parameters.RunPeak", "Threads": "$.Parameters.Threads", @@ -39,7 +40,8 @@ "Type": "ChocolateyInstallation", "Parameters": { "Scenario": "InstallChocolatey", - "PackageName": "chocolatey" + "PackageName": "chocolatey", + "SupportedPlatforms": "win-x64,win-arm64" } }, { @@ -47,7 +49,8 @@ "Parameters": { "Scenario": "InstallCompiler", "PackageName": "chocolatey", - "Packages": "cygwin" + "Packages": "cygwin", + "SupportedPlatforms": "win-x64,win-arm64" } }, { @@ -55,7 +58,7 @@ "Parameters": { "Scenario": "InstallCompiler", "CompilerVersion": "$.Parameters.CompilerVersion", - "CygwinPackages": "gcc-g++,gcc-fortran,gcc,libiconv-devel" + "CygwinPackages": "gcc-g++,gcc-fortran,gcc-core,libiconv-devel" } }, { diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPSPEED.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPSPEED.json index 0340ec0e15..5dc5ca29b2 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPSPEED.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPSPEED.json @@ -6,12 +6,12 @@ "RecommendedMinimumExecutionTime": "(4-cores)=02:00:00,(16-cores)=06:00:00,(64-cores)=12:00:00", "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows", - "Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24" + "Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24", + "Limitations": "The following benchmarks don't work on Windows: 607, 621, 627, 628, 638, 649" }, "Parameters": { "CompilerVersion": "", "Iterations": 2, - "SpecProfile": "fpspeed", "RunPeak": false, "Threads": "{LogicalCoreCount}", "Copies": "{LogicalCoreCount}", @@ -24,7 +24,8 @@ "Parameters": { "Scenario": "ExecuteSPECBenchmark", "Iterations": "$.Parameters.Iterations", - "SpecProfile": "$.Parameters.SpecProfile", + "SpecProfile": "fpspeed", + "Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"fpspeed\" : \"603 619 644 654\")}", "PackageName": "speccpu2017", "RunPeak": "$.Parameters.RunPeak", "Threads": "$.Parameters.Threads", @@ -39,7 +40,8 @@ "Type": "ChocolateyInstallation", "Parameters": { "Scenario": "InstallChocolatey", - "PackageName": "chocolatey" + "PackageName": "chocolatey", + "SupportedPlatforms": "win-x64,win-arm64" } }, { @@ -47,7 +49,8 @@ "Parameters": { "Scenario": "InstallCompiler", "PackageName": "chocolatey", - "Packages": "cygwin" + "Packages": "cygwin", + "SupportedPlatforms": "win-x64,win-arm64" } }, { @@ -55,7 +58,7 @@ "Parameters": { "Scenario": "InstallCompiler", "CompilerVersion": "$.Parameters.CompilerVersion", - "CygwinPackages": "gcc-g++,gcc-fortran,gcc,libiconv-devel" + "CygwinPackages": "gcc-g++,gcc-fortran,gcc-core,libiconv-devel" } }, { diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json index 8437825fdc..e4b4c676c6 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json @@ -6,12 +6,12 @@ "RecommendedMinimumExecutionTime": "(8-cores)=04:00:00,(16-cores)=08:00:00,(64-cores)=10:00:00", "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows", - "Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24" + "Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24", + "Limitations": "The following benchmarks don't work on Windows: 500, 502, 520, 523, 531" }, "Parameters": { "CompilerVersion": "", "Iterations": 2, - "SpecProfile": "intrate", "RunPeak": false, "Threads": "{LogicalCoreCount}", "Copies": "{LogicalCoreCount}", @@ -24,7 +24,8 @@ "Parameters": { "Scenario": "ExecuteSPECBenchmark", "Iterations": "$.Parameters.Iterations", - "SpecProfile": "$.Parameters.SpecProfile", + "SpecProfile": "intrate", + "Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"intrate\" : \"505 525 541 548 557\")}", "PackageName": "speccpu2017", "RunPeak": "$.Parameters.RunPeak", "Threads": "$.Parameters.Threads", @@ -39,7 +40,8 @@ "Type": "ChocolateyInstallation", "Parameters": { "Scenario": "InstallChocolatey", - "PackageName": "chocolatey" + "PackageName": "chocolatey", + "SupportedPlatforms": "win-x64,win-arm64" } }, { @@ -47,7 +49,8 @@ "Parameters": { "Scenario": "InstallCompiler", "PackageName": "chocolatey", - "Packages": "cygwin" + "Packages": "cygwin", + "SupportedPlatforms": "win-x64,win-arm64" } }, { @@ -55,7 +58,7 @@ "Parameters": { "Scenario": "InstallCompiler", "CompilerVersion": "$.Parameters.CompilerVersion", - "CygwinPackages": "gcc-g++,gcc-fortran,gcc,libiconv-devel" + "CygwinPackages": "gcc-g++,gcc-fortran,gcc-core,libiconv-devel" } }, { diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTSPEED.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTSPEED.json index ced218b3ce..39f3b04e9c 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTSPEED.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTSPEED.json @@ -6,12 +6,12 @@ "RecommendedMinimumExecutionTime": "(8-cores)=04:00:00,(16-cores)=06:00:00,(64-cores)=08:00:00", "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows", - "Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24" + "Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24", + "Limitations": "The following benchmarks don't work on Windows: 600, 602, 605, 620, 623, 625, 631, 648, 657" }, "Parameters": { "CompilerVersion": "", "Iterations": 2, - "SpecProfile": "intspeed", "RunPeak": false, "Threads": "{LogicalCoreCount}", "Copies": "{LogicalCoreCount}", @@ -24,7 +24,8 @@ "Parameters": { "Scenario": "ExecuteSPECBenchmark", "Iterations": "$.Parameters.Iterations", - "SpecProfile": "$.Parameters.SpecProfile", + "SpecProfile": "intspeed", + "Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"intspeed\" : \"641\")}", "PackageName": "speccpu2017", "RunPeak": "$.Parameters.RunPeak", "Threads": "$.Parameters.Threads", @@ -39,7 +40,8 @@ "Type": "ChocolateyInstallation", "Parameters": { "Scenario": "InstallChocolatey", - "PackageName": "chocolatey" + "PackageName": "chocolatey", + "SupportedPlatforms": "win-x64,win-arm64" } }, { @@ -47,7 +49,8 @@ "Parameters": { "Scenario": "InstallCompiler", "PackageName": "chocolatey", - "Packages": "cygwin" + "Packages": "cygwin", + "SupportedPlatforms": "win-x64,win-arm64" } }, { @@ -55,7 +58,7 @@ "Parameters": { "Scenario": "InstallCompiler", "CompilerVersion": "$.Parameters.CompilerVersion", - "CygwinPackages": "gcc-g++,gcc-fortran,gcc,libiconv-devel" + "CygwinPackages": "gcc-g++,gcc-fortran,gcc-core,libiconv-devel" } }, { diff --git a/website/docs/workloads/speccpu/speccpu-profiles.md b/website/docs/workloads/speccpu/speccpu-profiles.md index 9eb078a1c7..25c1affd2f 100644 --- a/website/docs/workloads/speccpu/speccpu-profiles.md +++ b/website/docs/workloads/speccpu/speccpu-profiles.md @@ -59,6 +59,9 @@ for evaluating the performance of the CPU for processing calculations. The following section provides a few basic examples of how to use the workload profile. Additional usage examples can be found in the 'Usage Scenarios/Examples' link at the top. +* * **Notes** + On Windows, only the following benchmarks are supported: 503.bwaves_r, 508.namd_r, 510.parest_r, 519.lbm_r, 544.nab_r, 549.fotonik3d_r, 554.roms_r. + ``` csharp # Execute the workload profile ./VirtualClient --profile=PERF-SPECCPU-FPRATE.json --system=Azure --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" @@ -110,6 +113,9 @@ for evaluating the performance of the CPU for processing calculations. * **Usage Examples** The following section provides a few basic examples of how to use the workload profile. +* **Notes** + On Windows, only the following benchmarks are supported: 603.bwaves_s, 619.lbm_s, 644.nab_s, 654.roms_s. + ``` bash # Execute the workload profile ./VirtualClient --profile=PERF-SPECCPU-FPSPEED.json --system=Azure --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" @@ -158,6 +164,9 @@ for evaluating the performance of the CPU for processing calculations. * **Usage Examples** The following section provides a few basic examples of how to use the workload profile. +* **Notes** + On Windows, only the following benchmarks are supported: 505.mcf_r, 525.x264_r, 541.leela_r, 548.exchange2_r, 557.xz_r. + ``` bash # Execute the workload profile ./VirtualClient --profile=PERF-SPECCPU-INTRATE.json --system=Azure --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" @@ -206,6 +215,9 @@ for evaluating the performance of the CPU for processing calculations. * **Usage Examples** The following section provides a few basic examples of how to use the workload profile. +* **Notes** + On Windows, only the following benchmarks are supported: 641.leela_s. + ``` bash # Execute the workload profile ./VirtualClient --profile=PERF-SPECCPU-INTSPEED.json --system=Azure --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" From bc55e26ce46a61c54a710190cb56833f5200ddaa Mon Sep 17 00:00:00 2001 From: saibulusu Date: Fri, 16 Jan 2026 16:53:12 -0800 Subject: [PATCH 2/3] minor documentation fix --- website/docs/workloads/speccpu/speccpu-profiles.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/docs/workloads/speccpu/speccpu-profiles.md b/website/docs/workloads/speccpu/speccpu-profiles.md index 25c1affd2f..fe1aaeb3d2 100644 --- a/website/docs/workloads/speccpu/speccpu-profiles.md +++ b/website/docs/workloads/speccpu/speccpu-profiles.md @@ -59,7 +59,7 @@ for evaluating the performance of the CPU for processing calculations. The following section provides a few basic examples of how to use the workload profile. Additional usage examples can be found in the 'Usage Scenarios/Examples' link at the top. -* * **Notes** +* **Notes** On Windows, only the following benchmarks are supported: 503.bwaves_r, 508.namd_r, 510.parest_r, 519.lbm_r, 544.nab_r, 549.fotonik3d_r, 554.roms_r. ``` csharp @@ -113,7 +113,7 @@ for evaluating the performance of the CPU for processing calculations. * **Usage Examples** The following section provides a few basic examples of how to use the workload profile. -* **Notes** +* **Notes** On Windows, only the following benchmarks are supported: 603.bwaves_s, 619.lbm_s, 644.nab_s, 654.roms_s. ``` bash @@ -164,7 +164,7 @@ for evaluating the performance of the CPU for processing calculations. * **Usage Examples** The following section provides a few basic examples of how to use the workload profile. -* **Notes** +* **Notes** On Windows, only the following benchmarks are supported: 505.mcf_r, 525.x264_r, 541.leela_r, 548.exchange2_r, 557.xz_r. ``` bash @@ -215,7 +215,7 @@ for evaluating the performance of the CPU for processing calculations. * **Usage Examples** The following section provides a few basic examples of how to use the workload profile. -* **Notes** +* **Notes** On Windows, only the following benchmarks are supported: 641.leela_s. ``` bash From 6ee05e2d7327e22c60cdbd7c8652f20c29e98c9a Mon Sep 17 00:00:00 2001 From: saibulusu Date: Fri, 16 Jan 2026 17:10:56 -0800 Subject: [PATCH 3/3] Fixing some test cases --- .../SPEC/SpecCpuExecutorTests.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs index 9b68802add..9fa817a77c 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs @@ -164,6 +164,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles this.mockFixture.Parameters = new Dictionary() { { nameof(SpecCpuExecutor.SpecProfile), "fprate" }, + { nameof(SpecCpuExecutor.Benchmarks), "fprate" }, { nameof(SpecCpuExecutor.PackageName), "speccpu" }, { nameof(SpecCpuExecutor.RunPeak), false }, }; @@ -200,6 +201,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles this.mockFixture.Parameters = new Dictionary() { { nameof(SpecCpuExecutor.SpecProfile), "intspeed" }, + { nameof(SpecCpuExecutor.Benchmarks), "intspeed" }, { nameof(SpecCpuExecutor.PackageName), "speccpu" }, { nameof(SpecCpuExecutor.RunPeak), true } }; @@ -238,6 +240,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles this.mockFixture.Parameters = new Dictionary() { { nameof(SpecCpuExecutor.SpecProfile), "intspeed" }, + { nameof(SpecCpuExecutor.Benchmarks), "intspeed" }, { nameof(SpecCpuExecutor.PackageName), "speccpu" }, { nameof(SpecCpuExecutor.Iterations), 1 }, { nameof(SpecCpuExecutor.RunPeak), true } @@ -280,6 +283,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles this.mockFixture.Parameters = new Dictionary() { { nameof(SpecCpuExecutor.SpecProfile), "fprate" }, + { nameof(SpecCpuExecutor.Benchmarks), "fprate" }, { nameof(SpecCpuExecutor.PackageName), "speccpu" }, { nameof(SpecCpuExecutor.RunPeak), false }, }; @@ -317,6 +321,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles this.mockFixture.Parameters = new Dictionary() { { nameof(SpecCpuExecutor.SpecProfile), "intspeed" }, + { nameof(SpecCpuExecutor.Benchmarks), "intspeed" }, { nameof(SpecCpuExecutor.PackageName), "speccpu" }, { nameof(SpecCpuExecutor.RunPeak), true } }; @@ -374,6 +379,7 @@ private void SetupLinux() this.mockFixture.Parameters = new Dictionary() { { nameof(SpecCpuExecutor.SpecProfile), "intrate" }, + { nameof(SpecCpuExecutor.Benchmarks), "intrate" }, { nameof(SpecCpuExecutor.PackageName), "speccpu" }, { nameof(SpecCpuExecutor.RunPeak), true }, { nameof(SpecCpuExecutor.Threads), 8 }, @@ -405,6 +411,7 @@ private void SetupWindows() this.mockFixture.Parameters = new Dictionary() { { nameof(SpecCpuExecutor.SpecProfile), "intrate" }, + { nameof(SpecCpuExecutor.Benchmarks), "intrate" }, { nameof(SpecCpuExecutor.PackageName), "speccpu" }, { nameof(SpecCpuExecutor.RunPeak), true }, { nameof(SpecCpuExecutor.Threads), 8 },