Skip to content

Speccpu limitations on Windows & specifying benchmarks in SPECCpuExecutor #619

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
saibulusu merged 3 commits into from
Jan 17, 2026
Merged

Speccpu limitations on Windows & specifying benchmarks in SPECCpuExecutor #619

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
635915e
speccpu limitations on windows, specifying benchmarks for SpeccpuExec…
Jan 17, 2026
bc55e26
minor documentation fix
Jan 17, 2026
6ee05e2
Fixing some test cases
Jan 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
{
{ nameof(SpecCpuExecutor.SpecProfile), "fprate" },
{ nameof(SpecCpuExecutor.Benchmarks), "fprate" },
{ nameof(SpecCpuExecutor.PackageName), "speccpu" },
{ nameof(SpecCpuExecutor.RunPeak), false },
};
Expand Down Expand Up @@ -200,6 +201,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
{
{ nameof(SpecCpuExecutor.SpecProfile), "intspeed" },
{ nameof(SpecCpuExecutor.Benchmarks), "intspeed" },
{ nameof(SpecCpuExecutor.PackageName), "speccpu" },
{ nameof(SpecCpuExecutor.RunPeak), true }
};
Expand Down Expand Up @@ -238,6 +240,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
{
{ nameof(SpecCpuExecutor.SpecProfile), "intspeed" },
{ nameof(SpecCpuExecutor.Benchmarks), "intspeed" },
{ nameof(SpecCpuExecutor.PackageName), "speccpu" },
{ nameof(SpecCpuExecutor.Iterations), 1 },
{ nameof(SpecCpuExecutor.RunPeak), true }
Expand Down Expand Up @@ -280,6 +283,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
{
{ nameof(SpecCpuExecutor.SpecProfile), "fprate" },
{ nameof(SpecCpuExecutor.Benchmarks), "fprate" },
{ nameof(SpecCpuExecutor.PackageName), "speccpu" },
{ nameof(SpecCpuExecutor.RunPeak), false },
};
Expand Down Expand Up @@ -317,6 +321,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
{
{ nameof(SpecCpuExecutor.SpecProfile), "intspeed" },
{ nameof(SpecCpuExecutor.Benchmarks), "intspeed" },
{ nameof(SpecCpuExecutor.PackageName), "speccpu" },
{ nameof(SpecCpuExecutor.RunPeak), true }
};
Expand Down Expand Up @@ -374,6 +379,7 @@ private void SetupLinux()
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
{
{ nameof(SpecCpuExecutor.SpecProfile), "intrate" },
{ nameof(SpecCpuExecutor.Benchmarks), "intrate" },
{ nameof(SpecCpuExecutor.PackageName), "speccpu" },
{ nameof(SpecCpuExecutor.RunPeak), true },
{ nameof(SpecCpuExecutor.Threads), 8 },
Expand Down Expand Up @@ -405,6 +411,7 @@ private void SetupWindows()
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
{
{ nameof(SpecCpuExecutor.SpecProfile), "intrate" },
{ nameof(SpecCpuExecutor.Benchmarks), "intrate" },
{ nameof(SpecCpuExecutor.PackageName), "speccpu" },
{ nameof(SpecCpuExecutor.RunPeak), true },
{ nameof(SpecCpuExecutor.Threads), 8 },
Expand Down
14 changes: 13 additions & 1 deletion src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuExecutor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,18 @@ public string SpecProfile
}
}

/// <summary>
/// List of benchmarks to run.
/// </summary>
public string Benchmarks
{
get
{
this.Parameters.TryGetValue(nameof(SpecCpuExecutor.Benchmarks), out IConvertible benchmarks);
return benchmarks?.ToString();
}
}

/// <summary>
/// The whether SPECcpu runs base tuning or base+peak tuning.
/// </summary>
Expand Down Expand Up @@ -396,7 +408,7 @@ private string GetCommandLineArguments()
// Iterations has to be either 2 or 3 for reportable runs. https://www.spec.org/cpu2017/Docs/config.html#reportable
bool reportable = (this.Platform == PlatformID.Unix) && (this.Iterations == 2 || this.Iterations == 3);
cmd = reportable ? $"{cmd} --reportable" : $"{cmd} --noreportable";
cmd = $"{cmd} {this.SpecProfile}";
cmd = $"{cmd} {this.Benchmarks}";
return cmd;
}

Expand Down
16 changes: 16 additions & 0 deletions src/VirtualClient/VirtualClient.Core.UnitTests/ProfileExpressionEvaluatorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1440,6 +1440,22 @@ public async Task ProfileExpressionEvaluatorSupportsTernaryFunctionReferencesInP
Assert.AreEqual(true, parameters["IsTLSEnabled"]);
}

[Test]
public async Task ProfileExpressionEvaluatorSupportsTernaryFunctionReferencesInParameterSets_Scenario_7()
{
this.SetupDefaults(PlatformID.Win32NT);

Dictionary<string, IConvertible> parameters = new Dictionary<string, IConvertible>
{
{ "Benchmarks" , "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"intrate\" : \"505 525 541 548 557\")}" },
};

await ProfileExpressionEvaluator.Instance.EvaluateAsync(this.mockFixture.Dependencies, parameters);

Assert.AreEqual("505 525 541 548 557", parameters["Benchmarks"]);
}


[Test]

public async Task ProfileExpressionEvaluatorSupportsNestedTernaryFunctionReferencesInParameterSets_Scenario_1()
Expand Down
15 changes: 9 additions & 6 deletions src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@
"RecommendedMinimumExecutionTime": "(4-cores)=02:00:00,(16-cores)=05:00:00,(64-cores)=10:00:00",
"SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64",
"SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows",
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24"
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24",
"Limitations": "The following benchmarks don't work on Windows: 507, 511, 521, 526, 527, 538"
},
"Parameters": {
"CompilerVersion": "",
"Iterations": 2,
"SpecProfile": "fprate",
"RunPeak": false,
"Threads": "{LogicalCoreCount}",
"Copies": "{LogicalCoreCount}",
Expand All @@ -24,7 +24,8 @@
"Parameters": {
"Scenario": "ExecuteSPECBenchmark",
"Iterations": "$.Parameters.Iterations",
"SpecProfile": "$.Parameters.SpecProfile",
"SpecProfile": "fprate",
"Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"fprate\" : \"503 508 510 519 544 549 554\")}",
"PackageName": "speccpu2017",
"RunPeak": "$.Parameters.RunPeak",
"Threads": "$.Parameters.Threads",
Expand All @@ -39,23 +40,25 @@
"Type": "ChocolateyInstallation",
"Parameters": {
"Scenario": "InstallChocolatey",
"PackageName": "chocolatey"
"PackageName": "chocolatey",
"SupportedPlatforms": "win-x64,win-arm64"
}
},
{
"Type": "ChocolateyPackageInstallation",
"Parameters": {
"Scenario": "InstallCompiler",
"PackageName": "chocolatey",
"Packages": "cygwin"
"Packages": "cygwin",
"SupportedPlatforms": "win-x64,win-arm64"
}
},
{
"Type": "CompilerInstallation",
"Parameters": {
"Scenario": "InstallCompiler",
"CompilerVersion": "$.Parameters.CompilerVersion",
"CygwinPackages": "gcc-g++,gcc-fortran,gcc,libiconv-devel"
"CygwinPackages": "gcc-g++,gcc-fortran,gcc-core,libiconv-devel"
}
},
{
Expand Down
15 changes: 9 additions & 6 deletions src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPSPEED.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@
"RecommendedMinimumExecutionTime": "(4-cores)=02:00:00,(16-cores)=06:00:00,(64-cores)=12:00:00",
"SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64",
"SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows",
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24"
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24",
"Limitations": "The following benchmarks don't work on Windows: 607, 621, 627, 628, 638, 649"
},
"Parameters": {
"CompilerVersion": "",
"Iterations": 2,
"SpecProfile": "fpspeed",
"RunPeak": false,
"Threads": "{LogicalCoreCount}",
"Copies": "{LogicalCoreCount}",
Expand All @@ -24,7 +24,8 @@
"Parameters": {
"Scenario": "ExecuteSPECBenchmark",
"Iterations": "$.Parameters.Iterations",
"SpecProfile": "$.Parameters.SpecProfile",
"SpecProfile": "fpspeed",
"Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"fpspeed\" : \"603 619 644 654\")}",
"PackageName": "speccpu2017",
"RunPeak": "$.Parameters.RunPeak",
"Threads": "$.Parameters.Threads",
Expand All @@ -39,23 +40,25 @@
"Type": "ChocolateyInstallation",
"Parameters": {
"Scenario": "InstallChocolatey",
"PackageName": "chocolatey"
"PackageName": "chocolatey",
"SupportedPlatforms": "win-x64,win-arm64"
}
},
{
"Type": "ChocolateyPackageInstallation",
"Parameters": {
"Scenario": "InstallCompiler",
"PackageName": "chocolatey",
"Packages": "cygwin"
"Packages": "cygwin",
"SupportedPlatforms": "win-x64,win-arm64"
}
},
{
"Type": "CompilerInstallation",
"Parameters": {
"Scenario": "InstallCompiler",
"CompilerVersion": "$.Parameters.CompilerVersion",
"CygwinPackages": "gcc-g++,gcc-fortran,gcc,libiconv-devel"
"CygwinPackages": "gcc-g++,gcc-fortran,gcc-core,libiconv-devel"
}
},
{
Expand Down
15 changes: 9 additions & 6 deletions src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@
"RecommendedMinimumExecutionTime": "(8-cores)=04:00:00,(16-cores)=08:00:00,(64-cores)=10:00:00",
"SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64",
"SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows",
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24"
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24",
"Limitations": "The following benchmarks don't work on Windows: 500, 502, 520, 523, 531"
},
"Parameters": {
"CompilerVersion": "",
"Iterations": 2,
"SpecProfile": "intrate",
"RunPeak": false,
"Threads": "{LogicalCoreCount}",
"Copies": "{LogicalCoreCount}",
Expand All @@ -24,7 +24,8 @@
"Parameters": {
"Scenario": "ExecuteSPECBenchmark",
"Iterations": "$.Parameters.Iterations",
"SpecProfile": "$.Parameters.SpecProfile",
"SpecProfile": "intrate",
"Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"intrate\" : \"505 525 541 548 557\")}",
"PackageName": "speccpu2017",
"RunPeak": "$.Parameters.RunPeak",
"Threads": "$.Parameters.Threads",
Expand All @@ -39,23 +40,25 @@
"Type": "ChocolateyInstallation",
"Parameters": {
"Scenario": "InstallChocolatey",
"PackageName": "chocolatey"
"PackageName": "chocolatey",
"SupportedPlatforms": "win-x64,win-arm64"
}
},
{
"Type": "ChocolateyPackageInstallation",
"Parameters": {
"Scenario": "InstallCompiler",
"PackageName": "chocolatey",
"Packages": "cygwin"
"Packages": "cygwin",
"SupportedPlatforms": "win-x64,win-arm64"
}
},
{
"Type": "CompilerInstallation",
"Parameters": {
"Scenario": "InstallCompiler",
"CompilerVersion": "$.Parameters.CompilerVersion",
"CygwinPackages": "gcc-g++,gcc-fortran,gcc,libiconv-devel"
"CygwinPackages": "gcc-g++,gcc-fortran,gcc-core,libiconv-devel"
}
},
{
Expand Down
15 changes: 9 additions & 6 deletions src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTSPEED.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@
"RecommendedMinimumExecutionTime": "(8-cores)=04:00:00,(16-cores)=06:00:00,(64-cores)=08:00:00",
"SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64",
"SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows",
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24"
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24",
"Limitations": "The following benchmarks don't work on Windows: 600, 602, 605, 620, 623, 625, 631, 648, 657"
},
"Parameters": {
"CompilerVersion": "",
"Iterations": 2,
"SpecProfile": "intspeed",
"RunPeak": false,
"Threads": "{LogicalCoreCount}",
"Copies": "{LogicalCoreCount}",
Expand All @@ -24,7 +24,8 @@
"Parameters": {
"Scenario": "ExecuteSPECBenchmark",
"Iterations": "$.Parameters.Iterations",
"SpecProfile": "$.Parameters.SpecProfile",
"SpecProfile": "intspeed",
"Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"intspeed\" : \"641\")}",
"PackageName": "speccpu2017",
"RunPeak": "$.Parameters.RunPeak",
"Threads": "$.Parameters.Threads",
Expand All @@ -39,23 +40,25 @@
"Type": "ChocolateyInstallation",
"Parameters": {
"Scenario": "InstallChocolatey",
"PackageName": "chocolatey"
"PackageName": "chocolatey",
"SupportedPlatforms": "win-x64,win-arm64"
}
},
{
"Type": "ChocolateyPackageInstallation",
"Parameters": {
"Scenario": "InstallCompiler",
"PackageName": "chocolatey",
"Packages": "cygwin"
"Packages": "cygwin",
"SupportedPlatforms": "win-x64,win-arm64"
}
},
{
"Type": "CompilerInstallation",
"Parameters": {
"Scenario": "InstallCompiler",
"CompilerVersion": "$.Parameters.CompilerVersion",
"CygwinPackages": "gcc-g++,gcc-fortran,gcc,libiconv-devel"
"CygwinPackages": "gcc-g++,gcc-fortran,gcc-core,libiconv-devel"
}
},
{
Expand Down
12 changes: 12 additions & 0 deletions website/docs/workloads/speccpu/speccpu-profiles.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ for evaluating the performance of the CPU for processing calculations.
The following section provides a few basic examples of how to use the workload profile. Additional usage examples can be found in the
'Usage Scenarios/Examples' link at the top.

* **Notes**
On Windows, only the following benchmarks are supported: 503.bwaves_r, 508.namd_r, 510.parest_r, 519.lbm_r, 544.nab_r, 549.fotonik3d_r, 554.roms_r.

``` csharp
# Execute the workload profile
./VirtualClient --profile=PERF-SPECCPU-FPRATE.json --system=Azure --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}"
Expand Down Expand Up @@ -110,6 +113,9 @@ for evaluating the performance of the CPU for processing calculations.
* **Usage Examples**
The following section provides a few basic examples of how to use the workload profile.

* **Notes**
On Windows, only the following benchmarks are supported: 603.bwaves_s, 619.lbm_s, 644.nab_s, 654.roms_s.

``` bash
# Execute the workload profile
./VirtualClient --profile=PERF-SPECCPU-FPSPEED.json --system=Azure --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}"
Expand Down Expand Up @@ -158,6 +164,9 @@ for evaluating the performance of the CPU for processing calculations.
* **Usage Examples**
The following section provides a few basic examples of how to use the workload profile.

* **Notes**
On Windows, only the following benchmarks are supported: 505.mcf_r, 525.x264_r, 541.leela_r, 548.exchange2_r, 557.xz_r.

``` bash
# Execute the workload profile
./VirtualClient --profile=PERF-SPECCPU-INTRATE.json --system=Azure --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}"
Expand Down Expand Up @@ -206,6 +215,9 @@ for evaluating the performance of the CPU for processing calculations.
* **Usage Examples**
The following section provides a few basic examples of how to use the workload profile.

* **Notes**
On Windows, only the following benchmarks are supported: 641.leela_s.

``` bash
# Execute the workload profile
./VirtualClient --profile=PERF-SPECCPU-INTSPEED.json --system=Azure --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}"
Expand Down
Loading