diff --git a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs
index 4ff5bf012a..d94087239c 100644
--- a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs
+++ b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs
@@ -1,4 +1,5 @@
using Perfolizer.Mathematics.SignificanceTesting;
+using Perfolizer.Mathematics.Thresholds;
namespace BenchmarkDotNet.Analysers
{
@@ -19,11 +20,11 @@ public static bool CheckZeroMeasurementOneSample(double[] results, double thresh
/// Checks distribution against Zero Measurement hypothesis in case of two samples
///
/// True if measurement is ZeroMeasurement
- public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead)
+ public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead, Threshold threshold = null)
{
if (workload.Length < 3 || overhead.Length < 3)
return false;
- return !WelchTest.Instance.IsGreater(workload, overhead).NullHypothesisIsRejected;
+ return !WelchTest.Instance.IsGreater(workload, overhead, threshold).NullHypothesisIsRejected;
}
}
}
\ No newline at end of file
diff --git a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs
index 21e7364a24..596c220645 100644
--- a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs
+++ b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs
@@ -14,9 +14,11 @@
[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)]
[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows,PublicKey=" + BenchmarkDotNetInfo.PublicKey)]
[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace,PublicKey=" + BenchmarkDotNetInfo.PublicKey)]
+[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning,PublicKey=" + BenchmarkDotNetInfo.PublicKey)]
#else
[assembly: InternalsVisibleTo("BenchmarkDotNet.Tests")]
[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests")]
[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows")]
[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace")]
+[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning")]
#endif
\ No newline at end of file
diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj
index 77e3aa5cff..b0259a941e 100755
--- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj
+++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj
@@ -16,7 +16,6 @@
-
@@ -26,6 +25,7 @@
+
diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs
new file mode 100644
index 0000000000..e36e43e0c3
--- /dev/null
+++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs
@@ -0,0 +1,235 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using BenchmarkDotNet.Analysers;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Portability;
+using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Tests.XUnit;
+using BenchmarkDotNet.Toolchains.InProcess.Emit;
+using Perfolizer.Horology;
+using Perfolizer.Mathematics.Thresholds;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace BenchmarkDotNet.IntegrationTests.ManualRunning
+{
+ public class ExpectedBenchmarkResultsTests : BenchmarkTestExecutor
+ {
+ // NativeAot takes a long time to build, so not including it in these tests.
+ // We also don't test InProcessNoEmitToolchain because it is known to be less accurate than code-gen toolchains.
+
+ private static readonly TimeInterval FallbackCpuResolutionValue = TimeInterval.FromNanoseconds(0.2d);
+
+ public ExpectedBenchmarkResultsTests(ITestOutputHelper output) : base(output) { }
+
+ private static IEnumerable EmptyBenchmarkTypes() =>
+ new[]
+ {
+ typeof(EmptyVoid),
+ typeof(EmptyByte),
+ typeof(EmptySByte),
+ typeof(EmptyShort),
+ typeof(EmptyUShort),
+ typeof(EmptyChar),
+ typeof(EmptyInt32),
+ typeof(EmptyUInt32),
+ typeof(EmptyInt64),
+ typeof(EmptyUInt64),
+ typeof(EmptyIntPtr),
+ typeof(EmptyUIntPtr),
+ typeof(EmptyVoidPointer),
+ typeof(EmptyClass)
+ };
+
+ public static IEnumerable