Add option to specify which summary stat you want the benchmark to return; defaults to . (#41)

Author: chriselrod

Project.toml

@@ -1,7 +1,7 @@
 name = "BLASBenchmarksCPU"
 uuid = "5fdc822c-4560-4d20-af7e-e5ee461714d5"
 authors = ["Chris Elrod <[email protected]> and contributors"]
-version = "0.2.0"
+version = "0.2.1"
 
 [deps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"

src/runbenchmark.jl

@@ -53,23 +53,32 @@ function maybe_sleep(x)
 end
 
 function benchmark_fun!(
-    f!::F, C, A, B, sleep_time, force_belapsed = false, reference = nothing
+    f!::F, summarystat, C, A, B, sleep_time, force_belapsed = false, reference = nothing
 ) where {F}
     maybe_sleep(sleep_time)
-    tmin = @elapsed f!(C, A, B)
+    t0 = @elapsed f!(C, A, B)
     isnothing(reference) || @assert C ≈ reference
-    if force_belapsed || 2tmin < BenchmarkTools.DEFAULT_PARAMETERS.seconds
+    if force_belapsed || 2t0 < BenchmarkTools.DEFAULT_PARAMETERS.seconds
         maybe_sleep(sleep_time)
-        tmin = min(tmin, @belapsed $f!($C, $A, $B))
-    else#if tmin < BenchmarkTools.DEFAULT_PARAMETERS.seconds
+        br = @benchmark $f!($C, $A, $B)
+        tret = summarystat(br).time
+        if summarystat === minimum # don't want to do this for `median` or `mean`, for example
+            tret = min(tret, t0)
+        end
+    else
         maybe_sleep(sleep_time)
-        tmin = min(tmin, @elapsed f!(C, A, B))
-        if tmin < 2BenchmarkTools.DEFAULT_PARAMETERS.seconds
+        t1 = @elapsed f!(C, A, B)
+        maybe_sleep(sleep_time)
+        t2 = @elapsed f!(C, A, B)
+        if (t0+t1) < 4BenchmarkTools.DEFAULT_PARAMETERS.seconds
             maybe_sleep(sleep_time)
-            tmin = min(tmin, @elapsed f!(C, A, B))
+            t3 = @elapsed f!(C, A, B)
+            tret = summarystat((t0, t1, t2, t3))
+        else
+            tret = summarystat((t0, t1, t2))
         end
     end
-    tmin
+    return tret
 end
 _mat_size(M, N, ::typeof(adjoint)) = (N, M)
 _mat_size(M, N, ::typeof(transpose)) = (N, M)
@@ -79,7 +88,6 @@ function alloc_mat(_M, _N, memory::Vector{T}, off, f = identity) where {T}
     A = f(reshape(view(memory, (off+1):(off+M*N)), (M, N)))
     A, off + align(M*N, T)
 end
-
 matmul_sizes(s::Integer) = (s,s,s)
 matmul_sizes(mkn::Tuple{Vararg{Integer,3}}) = mkn
 matmul_length(s) = prod(matmul_sizes(s))
@@ -174,14 +182,35 @@ function default_libs(::Type{T}) where {T}
     end
 end
 
+
+
 """
     runbench(T = Float64;
              libs = default_libs(T),
              sizes = logspace(2, 4000, 200),
              threaded::Bool = Threads.nthreads() > 1,
              A_transform = identity,
              B_transform = identity,
-             sleep_time = 0.0)
+             sleep_time = 0.0,
+             summarystat = median)
+
+ - T: The element type of the matrices.
+ - libs: Libraries to benchmark.
+ - sizes: Sizes of matrices to benchmark. Must be an iterable with either
+          `eltype(sizes) === Int` or `eltype(sizes) === NTuple{3,Int}`.
+          If the former, the matrices are square, with each dimension equal to the value.
+          If `i::NTuple{3,Int}`, it benchmarks `C = A * B` where `A` is `i[1]` by `i[2]`,
+            `B` is `i[2]` by `i[3]` and `C` is `i[1]` by `i[3]`.
+ - threaded: Should it benchmark multithreaded implementations?
+ - A_transform: a function to apply to `A`. Defaults to `identity`, but can be `adjoint`.
+ - B_transofrm: a function to apply to `B`. Defaults to `identity`, but can be `adjoint`.
+ - sleep_time: The use of this keyword argument is discouraged. If set, it will call `sleep`
+       in between benchmarks, the idea being to help keep the CPU cool. This is an unreliable
+       means of trying to get more reliable benchmarks. Instead, it's reccommended you disable
+       your systems turbo. Disabling it -- and reenabling when you're done benchmarking -- 
+       should be possible without requiring a reboot.
+  - summarystat: Which summary statistic should be reported? Defaults to `minimum`
+
 """
 function runbench(
     ::Type{T} = Float64;
@@ -190,7 +219,8 @@ function runbench(
     threaded::Bool = Threads.nthreads() > 1,
     A_transform = identity,
     B_transform = identity,
-    sleep_time = 0.0
+    sleep_time = 0.0,
+    summarystat = minimum
 ) where {T}
     if threaded
         mkl_set_num_threads(num_cores())
@@ -230,7 +260,7 @@ function runbench(
         for i ∈ eachindex(funcs)
             C, ref = i == 1 ? (C0, nothing) : (fill!(C1,junk(T)), C0)
             t = benchmark_fun!(
-                funcs[i], C, A, B, sleep_time, force_belapsed, ref
+                funcs[i], summarystat, C, A, B, sleep_time, force_belapsed, ref
             )
             gflops = 2e-9M*K*N / t
             times[j,i] = t

test/interface.jl

@@ -2,7 +2,7 @@
 import BLASBenchmarksCPU
 import StatsPlots
 @testset "Interface" begin
-    benchmark_result = BLASBenchmarksCPU.runbench(Float64; sizes = [1, 2, 5, 10, 20, 50, 100, 200], threaded=false) #test that threads=false at least doesn't throw somewhere.
+    benchmark_result = BLASBenchmarksCPU.runbench(Float64; sizes = [1, 2, 5, 10, 20, 50, 100, 200], threaded=false, summarystat = BLASBenchmarksCPU.median) #test that threads=false at least doesn't throw somewhere.
     df = BLASBenchmarksCPU.benchmark_result_df(benchmark_result)
     @test df isa BLASBenchmarksCPU.DataFrame
     df[!, :Size] = Float64.(df[!, :Size]);