Benchmark result df (#40)

* Fix benchmark_result_df

* Stack the DataFrame

* Changed the layout of the data frame

* Add interface test using

* threads -> threaded in test

Author: chriselrod

Project.toml

@@ -39,8 +39,9 @@ julia = "1.5"
 
 [extras]
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+StatsPlots = "f3b207a7-027a-5e70-b257-86293d7955fd"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 VectorizationBase = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f"
 
 [targets]
-test = ["InteractiveUtils", "Test", "VectorizationBase"]
+test = ["InteractiveUtils", "StatsPlots", "Test", "VectorizationBase"]

src/benchconfig.jl

@@ -26,11 +26,11 @@ function getfuncs(libs::Vector{Symbol}, threaded::Bool)::Vector{Function}
         elseif i === :BLIS || i === :blis
             gemmblis!
         elseif i === :Octavian
-            matmul!
+            threaded ? matmul! : matmul_serial!
         elseif i === :Tullio
             threaded ? tmul_threads! : tmul_no_threads!
         elseif i === :Gaius
-            Gaius.mul!
+            threaded ? Gaius.mul! : Gaius.mul_serial!
         elseif i === :generic || i === :Generic || i === :GENERIC
             generic_matmul!
         else

src/runbenchmark.jl

@@ -13,11 +13,26 @@ function benchmark_result_type(::BenchmarkResult{T}) where {T}
     return T
 end
 
+function _benchmark_result_df(sizes, libraries, mat)
+    df = DataFrame(Size = sizes)
+    for i ∈ eachindex(libraries)
+        setproperty!(df, libraries[i], mat[:,i])
+    end
+    return df
+end
+function _benchmark_result_df(br::BenchmarkResult, s::Symbol = :gflops)
+    _benchmark_result_df(br.sizes, br.libraries, getproperty(br, s))
+end
+
+
 """
     benchmark_result_df(benchmark_result::BenchmarkResult)
 """
 function benchmark_result_df(benchmark_result::BenchmarkResult)
-    return deepcopy(benchmark_result.df)
+    df = _benchmark_result_df(benchmark_result, :times)
+    df = stack(df, Not(:Size), variable_name = :Library, value_name = :Seconds)
+    df.GFLOPS = @. 2e-9 * matmul_length(df.Size) ./ df.Seconds
+    return df
 end
 
 """
@@ -29,10 +44,7 @@ end
 
 function Base.show(io::IO, br::BenchmarkResult{T}) where {T}
     println(io, "Bennchmark Result of Matrix{$T}, threaded = $(br.threaded)")
-    df = DataFrame(Sizes = br.sizes)
-    for i ∈ eachindex(br.libraries)
-        setproperty!(df, br.libraries[i], br.gflops[:,i])
-    end
+    df = _benchmark_result_df(br)
     println(io, df)
 end
 
@@ -70,6 +82,7 @@ end
 
 matmul_sizes(s::Integer) = (s,s,s)
 matmul_sizes(mkn::Tuple{Vararg{Integer,3}}) = mkn
+matmul_length(s) = prod(matmul_sizes(s))
 
 junk(::Type{T}) where {T <: Integer} = typemax(T) >> 1
 junk(::Type{T}) where {T} = T(NaN)

test/interface.jl

@@ -0,0 +1,13 @@
+
+import BLASBenchmarksCPU
+import StatsPlots
+@testset "Interface" begin
+    benchmark_result = BLASBenchmarksCPU.runbench(Float64; sizes = [1, 2, 5, 10, 20, 50, 100, 200], threaded=false) #test that threads=false at least doesn't throw somewhere.
+    df = BLASBenchmarksCPU.benchmark_result_df(benchmark_result)
+    @test df isa BLASBenchmarksCPU.DataFrame
+    df[!, :Size] = Float64.(df[!, :Size]);
+    df[!, :GFLOPS] = Float64.(df[!, :GFLOPS]);
+    df[!, :Seconds] = Float64.(df[!, :Seconds]);
+    p = StatsPlots.@df df StatsPlots.plot(:Size, :GFLOPS; group = :Library, legend = :bottomright)
+    @test p isa StatsPlots.Plots.Plot
+end

test/main.jl

@@ -13,6 +13,8 @@ for T in [Float64, Float32]
     )
     @test benchmark_result isa BLASBenchmarksCPU.BenchmarkResult
     @test benchmark_result_type(benchmark_result) === T
+    df = benchmark_result_df(benchmark_result)
+    @test df isa BLASBenchmarksCPU.DataFrame
     plot_directory = mktempdir()
     BLASBenchmarksCPU.plot(
         benchmark_result;

test/runtests.jl

@@ -9,3 +9,5 @@ include("test-suite-preamble.jl")
 @info("VectorizationBase.num_cores() is $(VectorizationBase.num_cores())")
 
 include("main.jl")
+include("interface.jl")
+