From 3e7da851ea9caa0e267c21e0bb067ae32ee9ad77 Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Thu, 6 Dec 2018 11:18:45 -0500
Subject: [PATCH 01/15] Add Decimal Representations Comparison benchmark.

Adds a benchmark file that produces performance comparisons across
various types and operations.
---
 bench/decimal-representation-comparisons.jl | 122 ++++++++++++++++++++
 1 file changed, 122 insertions(+)
 create mode 100644 bench/decimal-representation-comparisons.jl

diff --git a/bench/decimal-representation-comparisons.jl b/bench/decimal-representation-comparisons.jl
new file mode 100644
index 0000000..44587ef
--- /dev/null
+++ b/bench/decimal-representation-comparisons.jl
@@ -0,0 +1,122 @@
+# Decimal Representation Comparisons
+#
+# This benchmark compares the performance of several numeric representations, over various
+# numeric operations (+,-,*,/,÷...) on large arrays of numbers, in order to guide
+# decision-making about how to represent fixed-decimal numbers.
+#
+# It compares fixed-decimal types against the builtin Int and Float types of various sizes.
+# The output is written to a .csv file in the same directory as this file.
+
+module DecimalRepresentationComparisons
+
+using FixedPointDecimals
+using Random
+using BenchmarkTools, Statistics
+using DataFrames
+using CSV
+
+decimal_precision = 2
+
+# Express that data through the various types. Round it for integers.
+fd_FixedPointDecimal_types = [
+    FixedPointDecimals.FixedDecimal{Int32, decimal_precision},
+    FixedPointDecimals.FixedDecimal{Int64, decimal_precision},
+    FixedPointDecimals.FixedDecimal{Int128, decimal_precision},
+]
+inttypes = [Int32,Int64,Int128]
+floattypes = [Float32,Float64]
+bigtypes = [BigInt, BigFloat]
+
+alltypes = (inttypes..., bigtypes..., floattypes..., fd_FixedPointDecimal_types...,)
+
+identity1(a,_) = a
+allops = (*, /, +, ÷, identity1)
+
+# Category for the results output CSV
+category(::Type{<:Union{inttypes...}}) = "Int"
+category(::Type{<:Union{floattypes...}}) = "Float"
+category(::Type{<:Union{bigtypes...}}) = "Big"
+category(::Type{<:FixedPointDecimals.FixedDecimal}) = "FixedDecimal"
+type(T::Type) = "$T"
+type(T::Type{<:Union{Int32, Int64}}) = "  $T"
+type(T::Type{Int128}) = " $T"
+type(::Type{FixedPointDecimals.FixedDecimal{T,f}}) where {T,f} = "FD{$T,$f}"
+type(::Type{FixedPointDecimals.FixedDecimal{T,f}}) where {T<:Union{Int32,Int64},f} = "FD{ $T,$f}"
+opname(f) = Symbol(f)
+opname(f::typeof(identity1)) = :identity
+
+# --------- Define benchmark functions -------------
+# Some care is taken here to prevent the compiler from optimizing away the operations:
+#  - Marked @noinline so the constants we pass in aren't available to the optimizer.
+#  - We take `a` and `out` as parameters so that their values aren't available when
+#      compiling this function.
+#  - `out` is a Ref{T} so that this function will have side effects. We use an output
+#      parameter instead of returning the value directly  so that it will play nicely with
+#      the `@benchmark` macro which returns the benchmark results as an object.
+#  - `T` and `op` _should_ be available as compile-time constants, since we don't want to be
+#      measuring the time it takes to read from global variables.
+@noinline function benchmark(::Type{T}, op, a::T, n, out::Ref{T}) where {T}
+    for _ in 1:n
+        tmp = op(a,a)
+        out[] += tmp
+        a += one(T)
+    end
+end
+
+@noinline function baseline(::Type{T}, a::T, n, out::Ref{T}) where {T}
+    for _ in 1:n
+        tmp = a
+        out[] += tmp
+        a += one(T)
+    end
+end
+
+# ------------ Run the Benchmarks -------------------------
+function perform_benchmark()
+    # Collect the results
+    results = DataFrame(Operation=Symbol[], Category=String[], Type=String[],
+                        DurationNs=Float64[], Allocations=Int[], MinGcTime=Number[],
+                        Value=Number[])
+
+    # Run the benchmarks
+    for op in allops
+        println("$op")
+        for T in alltypes
+            print("$T ")
+
+            N = 1_000_000
+            initial_value = zero(T)
+            a = one(T)
+
+            # For some reason this is necessary to eliminate mysterious "1 allocation"
+            fbase = @eval (out::Ref{$T})->baseline($T, $a, $N, out)
+            fbench = @eval (out::Ref{$T})->benchmark($T, $op, $a, $N, out)
+
+            # Run the benchmark
+            outbase = Ref(initial_value)
+            bbase = median(@benchmark $fbase($outbase) evals=1 setup=($outbase[]=$initial_value))
+            outbench = Ref(initial_value)
+            bbench = median(@benchmark $fbench($outbench) evals=1 setup=($outbench[]=$initial_value))
+
+            # Compute results
+            difftime = (bbench.time - bbase.time)
+            println("$(round(difftime, digits=2)) ns ($(bbench.allocs) allocations)")
+            println(outbench[])
+            println(outbase[])
+            value = outbench
+
+            push!(results, Dict(:Operation=>opname(op), :Category=>category(T), :Type=>type(T),
+                                :DurationNs=>difftime/N, # average (b.times reports ns)
+                                :Allocations=>bbench.allocs, :MinGcTime=>bbench.gctime,
+                                :Value=>value[]))
+        end
+    end
+
+    println(results)
+    CSV.write("$(@__DIR__)/comparisons-benchmark-results.csv", results)
+    return results
+end
+
+results = perform_benchmark()
+
+end

From bc4083fb7c611a295a563a67cb1091d7e8b5f914 Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Mon, 10 Dec 2018 13:24:52 -0500
Subject: [PATCH 02/15] Move bench to benchmark/benchmarks.jl for
 PkgBenchmark.jl

---
 .../benchmarks.jl                                                 | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename bench/decimal-representation-comparisons.jl => benchmark/benchmarks.jl (100%)

diff --git a/bench/decimal-representation-comparisons.jl b/benchmark/benchmarks.jl
similarity index 100%
rename from bench/decimal-representation-comparisons.jl
rename to benchmark/benchmarks.jl

From 75639c6800dc5e766fcdf966de5361194f532c8b Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Tue, 11 Dec 2018 13:20:35 -0500
Subject: [PATCH 03/15] Trying to switch to use PkgBenchmark.jl

---
 benchmark/REQUIRE                |  1 +
 benchmark/benchmarks.jl          | 87 +++++++++++++-------------------
 benchmark/subtract-benchmarks.jl | 10 ++++
 3 files changed, 47 insertions(+), 51 deletions(-)
 create mode 100644 benchmark/REQUIRE
 create mode 100644 benchmark/subtract-benchmarks.jl

diff --git a/benchmark/REQUIRE b/benchmark/REQUIRE
new file mode 100644
index 0000000..bf369b9
--- /dev/null
+++ b/benchmark/REQUIRE
@@ -0,0 +1 @@
+BenchmarkTools
diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index 44587ef..0595f68 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -7,13 +7,16 @@
 # It compares fixed-decimal types against the builtin Int and Float types of various sizes.
 # The output is written to a .csv file in the same directory as this file.
 
-module DecimalRepresentationComparisons
-
 using FixedPointDecimals
 using Random
 using BenchmarkTools, Statistics
-using DataFrames
-using CSV
+
+# # TODO: remove this file once BenchmarkTools has a built-in solution for diffing two
+# # @benchmarkable runs
+# include("subtract-benchmarks.jl")
+
+# Define a parent BenchmarkGroup to contain our suite
+const SUITE = BenchmarkGroup()
 
 decimal_precision = 2
 
@@ -42,8 +45,8 @@ type(T::Type{<:Union{Int32, Int64}}) = "  $T"
 type(T::Type{Int128}) = " $T"
 type(::Type{FixedPointDecimals.FixedDecimal{T,f}}) where {T,f} = "FD{$T,$f}"
 type(::Type{FixedPointDecimals.FixedDecimal{T,f}}) where {T<:Union{Int32,Int64},f} = "FD{ $T,$f}"
-opname(f) = Symbol(f)
-opname(f::typeof(identity1)) = :identity
+opname(f) = string(Symbol(f))
+opname(f::typeof(identity1)) = "identity"
 
 # --------- Define benchmark functions -------------
 # Some care is taken here to prevent the compiler from optimizing away the operations:
@@ -71,52 +74,34 @@ end
     end
 end
 
-# ------------ Run the Benchmarks -------------------------
-function perform_benchmark()
-    # Collect the results
-    results = DataFrame(Operation=Symbol[], Category=String[], Type=String[],
-                        DurationNs=Float64[], Allocations=Int[], MinGcTime=Number[],
-                        Value=Number[])
-
-    # Run the benchmarks
-    for op in allops
-        println("$op")
-        for T in alltypes
-            print("$T ")
-
-            N = 1_000_000
-            initial_value = zero(T)
-            a = one(T)
-
-            # For some reason this is necessary to eliminate mysterious "1 allocation"
-            fbase = @eval (out::Ref{$T})->baseline($T, $a, $N, out)
-            fbench = @eval (out::Ref{$T})->benchmark($T, $op, $a, $N, out)
-
-            # Run the benchmark
-            outbase = Ref(initial_value)
-            bbase = median(@benchmark $fbase($outbase) evals=1 setup=($outbase[]=$initial_value))
-            outbench = Ref(initial_value)
-            bbench = median(@benchmark $fbench($outbench) evals=1 setup=($outbench[]=$initial_value))
-
-            # Compute results
-            difftime = (bbench.time - bbase.time)
-            println("$(round(difftime, digits=2)) ns ($(bbench.allocs) allocations)")
-            println(outbench[])
-            println(outbase[])
-            value = outbench
-
-            push!(results, Dict(:Operation=>opname(op), :Category=>category(T), :Type=>type(T),
-                                :DurationNs=>difftime/N, # average (b.times reports ns)
-                                :Allocations=>bbench.allocs, :MinGcTime=>bbench.gctime,
-                                :Value=>value[]))
-        end
+# Define the benchmark structure
+for op in allops
+    SUITE[opname(op)] = BenchmarkGroup()
+    for T in alltypes
+        SUITE[opname(op)][type(T)] = BenchmarkGroup(["diff"])
     end
-
-    println(results)
-    CSV.write("$(@__DIR__)/comparisons-benchmark-results.csv", results)
-    return results
 end
 
-results = perform_benchmark()
-
+for op in allops
+    println()
+    println("$op")
+    for T in alltypes
+        print("$T ")
+
+        N = 1 # _000 #_000
+        initial_value = zero(T)
+        a = one(T)
+
+        # For some reason this is necessary to eliminate mysterious "1 allocation"
+        fbase = @eval (out::Ref{$T})->baseline($T, $a, $N, out)
+        fbench = @eval (out::Ref{$T})->benchmark($T, $op, $a, $N, out)
+
+        # Run the benchmark
+        outbase = Ref(initial_value)
+        bbase = @benchmarkable $fbase($outbase) evals=1 setup=($outbase[]=$initial_value)
+        outbench = Ref(initial_value)
+        bbench = @benchmarkable $fbench($outbench) evals=1 setup=($outbench[]=$initial_value)
+        bdiff = bbench - bbase
+        SUITE[opname(op)][type(T)]["diff"] = bdiff
+    end
 end
diff --git a/benchmark/subtract-benchmarks.jl b/benchmark/subtract-benchmarks.jl
new file mode 100644
index 0000000..1338f7f
--- /dev/null
+++ b/benchmark/subtract-benchmarks.jl
@@ -0,0 +1,10 @@
+import PkgBenchmark
+function PkgBenchmark._run(b::PkgBenchmark.BenchmarkTools.BenchmarkDiff, p::PkgBenchmark.BenchmarkTools.Parameters = b.params;
+    prog = nothing, verbose::Bool = false, pad = "", hierarchy = [], kwargs...)
+    res = BenchmarkTools.run_result(b, p; kwargs...)[1]
+    if prog != nothing
+        indent = 0
+        ProgressMeter.next!(prog; showvalues = [map(id -> ("  "^(indent += 1) * "[$(id[2])/$(id[3])]", id[1]), hierarchy)...])
+    end
+    return res
+end

From 54e4caefc767983e94316e248f5f97628fa4517d Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Tue, 11 Dec 2018 14:15:52 -0500
Subject: [PATCH 04/15] Add main script: benchmark/runbench.jl

Use custom branch of `PkgBenchmark.jl` to support post-processing, which
we need.
---
 benchmark/Manifest.toml | 104 ++++++++++++++++++++++++++++++++++++++++
 benchmark/Project.toml  |   3 ++
 benchmark/benchmarks.jl |  19 ++++----
 benchmark/runbench.jl   |  51 ++++++++++++++++++++
 4 files changed, 167 insertions(+), 10 deletions(-)
 create mode 100644 benchmark/Manifest.toml
 create mode 100644 benchmark/Project.toml
 create mode 100644 benchmark/runbench.jl

diff --git a/benchmark/Manifest.toml b/benchmark/Manifest.toml
new file mode 100644
index 0000000..e0afbdf
--- /dev/null
+++ b/benchmark/Manifest.toml
@@ -0,0 +1,104 @@
+[[Base64]]
+uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+
+[[BenchmarkTools]]
+deps = ["JSON", "Printf", "Statistics", "Test"]
+git-tree-sha1 = "e686f1754227e4748259f400839b83a1e8773e02"
+uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+version = "0.4.1"
+
+[[Dates]]
+deps = ["Printf"]
+uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
+
+[[Distributed]]
+deps = ["Random", "Serialization", "Sockets"]
+uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+
+[[InteractiveUtils]]
+deps = ["Markdown"]
+uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+
+[[JSON]]
+deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"]
+git-tree-sha1 = "1f7a25b53ec67f5e9422f1f551ee216503f4a0fa"
+uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+version = "0.20.0"
+
+[[LibGit2]]
+uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
+
+[[Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+
+[[LinearAlgebra]]
+deps = ["Libdl"]
+uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+
+[[Logging]]
+uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
+
+[[Markdown]]
+deps = ["Base64"]
+uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+
+[[Mmap]]
+uuid = "a63ad114-7e13-5084-954f-fe012c677804"
+
+[[Pkg]]
+deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
+uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+
+[[PkgBenchmark]]
+deps = ["BenchmarkTools", "Dates", "InteractiveUtils", "JSON", "LibGit2", "Pkg", "Printf", "ProgressMeter", "Random", "Test"]
+git-tree-sha1 = "d15f09c1e4a8c1075db6025fed97b8a852d5738e"
+repo-rev = "post-processing"
+repo-url = "https://github.com/NHDaly/PkgBenchmark.jl"
+uuid = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
+version = "0.2.0+"
+
+[[Printf]]
+deps = ["Unicode"]
+uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+
+[[ProgressMeter]]
+deps = ["Distributed", "Printf", "Random", "Test"]
+git-tree-sha1 = "48058bc11607676e5bbc0b974af79106c6200787"
+uuid = "92933f4c-e287-5a05-a399-4b506db050ca"
+version = "0.9.0"
+
+[[REPL]]
+deps = ["InteractiveUtils", "Markdown", "Sockets"]
+uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+
+[[Random]]
+deps = ["Serialization"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+
+[[SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+
+[[Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+
+[[Sockets]]
+uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
+
+[[SparseArrays]]
+deps = ["LinearAlgebra", "Random"]
+uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+
+[[Statistics]]
+deps = ["LinearAlgebra", "SparseArrays"]
+uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+
+[[Test]]
+deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
+uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[[UUIDs]]
+deps = ["Random", "SHA"]
+uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[[Unicode]]
+uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
diff --git a/benchmark/Project.toml b/benchmark/Project.toml
new file mode 100644
index 0000000..16acc49
--- /dev/null
+++ b/benchmark/Project.toml
@@ -0,0 +1,3 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index 0595f68..f8b267e 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -7,16 +7,18 @@
 # It compares fixed-decimal types against the builtin Int and Float types of various sizes.
 # The output is written to a .csv file in the same directory as this file.
 
+# TODO: remove this file once BenchmarkTools has a built-in solution for diffing two
+# @benchmarkable runs
+using Pkg
+Pkg.activate(@__DIR__)
+
 using FixedPointDecimals
 using Random
 using BenchmarkTools, Statistics
 
-# # TODO: remove this file once BenchmarkTools has a built-in solution for diffing two
-# # @benchmarkable runs
-# include("subtract-benchmarks.jl")
-
 # Define a parent BenchmarkGroup to contain our suite
 const SUITE = BenchmarkGroup()
+const N = parse(Int, get(ENV, "BENCH_NUM_ITERS", "1000"))
 
 decimal_precision = 2
 
@@ -78,7 +80,7 @@ end
 for op in allops
     SUITE[opname(op)] = BenchmarkGroup()
     for T in alltypes
-        SUITE[opname(op)][type(T)] = BenchmarkGroup(["diff"])
+        SUITE[opname(op)][type(T)] = BenchmarkGroup(["base", "bench"])
     end
 end
 
@@ -88,7 +90,6 @@ for op in allops
     for T in alltypes
         print("$T ")
 
-        N = 1 # _000 #_000
         initial_value = zero(T)
         a = one(T)
 
@@ -98,10 +99,8 @@ for op in allops
 
         # Run the benchmark
         outbase = Ref(initial_value)
-        bbase = @benchmarkable $fbase($outbase) evals=1 setup=($outbase[]=$initial_value)
+        SUITE[opname(op)][type(T)]["base"] = @benchmarkable $fbase($outbase) evals=1 setup=($outbase[]=$initial_value)
         outbench = Ref(initial_value)
-        bbench = @benchmarkable $fbench($outbench) evals=1 setup=($outbench[]=$initial_value)
-        bdiff = bbench - bbase
-        SUITE[opname(op)][type(T)]["diff"] = bdiff
+        SUITE[opname(op)][type(T)]["bench"] = @benchmarkable $fbench($outbench) evals=1 setup=($outbench[]=$initial_value)
     end
 end
diff --git a/benchmark/runbench.jl b/benchmark/runbench.jl
new file mode 100644
index 0000000..251179f
--- /dev/null
+++ b/benchmark/runbench.jl
@@ -0,0 +1,51 @@
+module FixedPointDecimals_RunBench
+
+using Pkg
+
+Pkg.activate(@__DIR__)
+using PkgBenchmark, BenchmarkTools, Statistics
+
+const N = 1_000
+
+import Base: -, /
+function -(a::BenchmarkTools.TrialEstimate, b::BenchmarkTools.TrialEstimate)
+    ttol = max(params(a).time_tolerance, params(b).time_tolerance)
+    mtol = max(params(a).memory_tolerance, params(b).memory_tolerance)
+    p = BenchmarkTools.Parameters(params(a); time_tolerance = ttol, memory_tolerance = mtol)
+    return BenchmarkTools.TrialEstimate(p, -(time(a), time(b)), -(gctime(a), gctime(b)),
+                                        -(memory(a), memory(b)), -(allocs(a), allocs(b)))
+end
+function /(a::BenchmarkTools.TrialEstimate, b::Int)
+    ttol = params(a).time_tolerance / b
+    mtol = params(a).memory_tolerance / b
+    p = BenchmarkTools.Parameters(params(a); time_tolerance = ttol, memory_tolerance = mtol)
+    return BenchmarkTools.TrialEstimate(p, time(a)/b, gctime(a)/b,
+                                        memory(a)/b, allocs(a)/b)
+end
+
+function postprocess(results::BenchmarkGroup)
+    global _results = deepcopy(results)
+    for (op, op_group) in results.data
+        op_results = op_group.data
+        for (type, type_group) in op_results
+            benchresults = type_group.data
+            if op == "identity"
+                # For :identity, bench and base are identical so we don't want to subtract.
+                op_results[type] = median(benchresults["bench"]) / N
+            else
+                op_results[type] = median(benchresults["bench"])/N - median(benchresults["base"])/N
+            end
+        end
+    end
+    results
+end
+results = deepcopy(_results)
+((results)->(results["*"]["BigInt"]["bench"]=median(results["*"]["BigInt"]["bench"])/1000))(results)
+
+bench_results = withenv("BENCH_NUM_ITERS"=>string(N)) do
+    benchmarkpkg("FixedPointDecimals"; postprocess=postprocess)
+end
+
+export_markdown("results.md", bench_results)
+
+end

From 0dbf53b838d20ddbbb613bc7a37eb5dd22baeab9 Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Thu, 20 Dec 2018 13:05:16 -0500
Subject: [PATCH 05/15] Cleanup cruft; update PkgBenchmark.jl commit

---
 benchmark/Manifest.toml | 4 ++--
 benchmark/runbench.jl   | 3 ---
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/benchmark/Manifest.toml b/benchmark/Manifest.toml
index e0afbdf..2716f24 100644
--- a/benchmark/Manifest.toml
+++ b/benchmark/Manifest.toml
@@ -50,8 +50,8 @@ deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUID
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 
 [[PkgBenchmark]]
-deps = ["BenchmarkTools", "Dates", "InteractiveUtils", "JSON", "LibGit2", "Pkg", "Printf", "ProgressMeter", "Random", "Test"]
-git-tree-sha1 = "d15f09c1e4a8c1075db6025fed97b8a852d5738e"
+deps = ["BenchmarkTools", "Dates", "InteractiveUtils", "JSON", "LibGit2", "Pkg", "Printf", "ProgressMeter", "Random", "Statistics", "Test"]
+git-tree-sha1 = "717f93d4cfcbd5b5dadbbb74071541c725ba6552"
 repo-rev = "post-processing"
 repo-url = "https://github.com/NHDaly/PkgBenchmark.jl"
 uuid = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
diff --git a/benchmark/runbench.jl b/benchmark/runbench.jl
index 251179f..dd115d7 100644
--- a/benchmark/runbench.jl
+++ b/benchmark/runbench.jl
@@ -24,7 +24,6 @@ function /(a::BenchmarkTools.TrialEstimate, b::Int)
 end
 
 function postprocess(results::BenchmarkGroup)
-    global _results = deepcopy(results)
     for (op, op_group) in results.data
         op_results = op_group.data
         for (type, type_group) in op_results
@@ -39,8 +38,6 @@ function postprocess(results::BenchmarkGroup)
     end
     results
 end
-results = deepcopy(_results)
-((results)->(results["*"]["BigInt"]["bench"]=median(results["*"]["BigInt"]["bench"])/1000))(results)
 
 bench_results = withenv("BENCH_NUM_ITERS"=>string(N)) do
     benchmarkpkg("FixedPointDecimals"; postprocess=postprocess)

From f6a48f859c4b159e8add13ad5e247148a5558372 Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Thu, 20 Dec 2018 13:37:41 -0500
Subject: [PATCH 06/15] Write `results.md` to `benchmark/` directory.

---
 benchmark/runbench.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmark/runbench.jl b/benchmark/runbench.jl
index dd115d7..4d4df5a 100644
--- a/benchmark/runbench.jl
+++ b/benchmark/runbench.jl
@@ -43,6 +43,6 @@ bench_results = withenv("BENCH_NUM_ITERS"=>string(N)) do
     benchmarkpkg("FixedPointDecimals"; postprocess=postprocess)
 end
 
-export_markdown("results.md", bench_results)
+export_markdown(joinpath(@__DIR__, "results.md"), bench_results)
 
 end

From 90a86b5df6b647f7de533caa4e64732ad4cdb55a Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Fri, 4 Jan 2019 15:37:58 -0500
Subject: [PATCH 07/15] Fix `/` in postproccess to `round()` mem and allocs

---
 benchmark/runbench.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmark/runbench.jl b/benchmark/runbench.jl
index 4d4df5a..aa1aa7a 100644
--- a/benchmark/runbench.jl
+++ b/benchmark/runbench.jl
@@ -20,7 +20,7 @@ function /(a::BenchmarkTools.TrialEstimate, b::Int)
     mtol = params(a).memory_tolerance / b
     p = BenchmarkTools.Parameters(params(a); time_tolerance = ttol, memory_tolerance = mtol)
     return BenchmarkTools.TrialEstimate(p, time(a)/b, gctime(a)/b,
-                                        memory(a)/b, allocs(a)/b)
+                                        round(memory(a)/b), round(allocs(a)/b))
 end
 
 function postprocess(results::BenchmarkGroup)

From b078194caba4eca0c9b5d0b525705ee5972e3abc Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Wed, 9 Jan 2019 21:58:34 -0500
Subject: [PATCH 08/15] Remove BigInt,BigFloat from benchmarks cause too slow

---
 benchmark/benchmarks.jl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index f8b267e..645c2eb 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -30,9 +30,9 @@ fd_FixedPointDecimal_types = [
 ]
 inttypes = [Int32,Int64,Int128]
 floattypes = [Float32,Float64]
-bigtypes = [BigInt, BigFloat]
+#bigtypes = [BigInt, BigFloat]
 
-alltypes = (inttypes..., bigtypes..., floattypes..., fd_FixedPointDecimal_types...,)
+alltypes = (inttypes..., floattypes..., fd_FixedPointDecimal_types...,)
 
 identity1(a,_) = a
 allops = (*, /, +, ÷, identity1)
@@ -40,7 +40,7 @@ allops = (*, /, +, ÷, identity1)
 # Category for the results output CSV
 category(::Type{<:Union{inttypes...}}) = "Int"
 category(::Type{<:Union{floattypes...}}) = "Float"
-category(::Type{<:Union{bigtypes...}}) = "Big"
+#category(::Type{<:Union{bigtypes...}}) = "Big"
 category(::Type{<:FixedPointDecimals.FixedDecimal}) = "FixedDecimal"
 type(T::Type) = "$T"
 type(T::Type{<:Union{Int32, Int64}}) = "  $T"

From c3ac10f7a5a60004bf959861f95394b79f5ef56f Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Sun, 27 Jan 2019 09:11:44 -0500
Subject: [PATCH 09/15] Switch to PkgBenchmark/master after merge

---
 benchmark/Manifest.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/benchmark/Manifest.toml b/benchmark/Manifest.toml
index 2716f24..f3b6d51 100644
--- a/benchmark/Manifest.toml
+++ b/benchmark/Manifest.toml
@@ -52,8 +52,8 @@ uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 [[PkgBenchmark]]
 deps = ["BenchmarkTools", "Dates", "InteractiveUtils", "JSON", "LibGit2", "Pkg", "Printf", "ProgressMeter", "Random", "Statistics", "Test"]
 git-tree-sha1 = "717f93d4cfcbd5b5dadbbb74071541c725ba6552"
-repo-rev = "post-processing"
-repo-url = "https://github.com/NHDaly/PkgBenchmark.jl"
+repo-rev = "master"
+repo-url = "https://github.com/JuliaCI/PkgBenchmark.jl.git"
 uuid = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
 version = "0.2.0+"
 

From a258785fb875cf933d0af0c4ec37b513a9917e0a Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Sun, 27 Jan 2019 09:24:45 -0500
Subject: [PATCH 10/15] Add `judgebench()` to judge perf b/w commits

---
 benchmark/runbench.jl | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/benchmark/runbench.jl b/benchmark/runbench.jl
index aa1aa7a..24c6221 100644
--- a/benchmark/runbench.jl
+++ b/benchmark/runbench.jl
@@ -39,10 +39,21 @@ function postprocess(results::BenchmarkGroup)
     results
 end
 
-bench_results = withenv("BENCH_NUM_ITERS"=>string(N)) do
-    benchmarkpkg("FixedPointDecimals"; postprocess=postprocess)
+function runbench()
+    bench_results = withenv("BENCH_NUM_ITERS"=>string(N)) do
+        benchmarkpkg("FixedPointDecimals"; postprocess=postprocess)
+    end
+
+    export_markdown(joinpath(@__DIR__, "results.md"), bench_results)
 end
 
-export_markdown(joinpath(@__DIR__, "results.md"), bench_results)
+function judgebench(target::Union{String, BenchmarkConfig}, baseline::Union{String, BenchmarkConfig})
+    bench_results = withenv("BENCH_NUM_ITERS"=>string(N)) do
+        judge("FixedPointDecimals", target, baseline; f=identity, postprocess=postprocess)
+    end
+end
+function judgebench(baseline::Union{String, BenchmarkConfig})
+    judgebench(BenchmarkConfig(), baseline)
+end
 
 end

From 8d6712ff71282ccee31ae1451a33fb556f7256c8 Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Sun, 27 Jan 2019 10:24:14 -0500
Subject: [PATCH 11/15] Don't divide by N when judging two commits to prevent
 noisiness

---
 benchmark/runbench.jl | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/benchmark/runbench.jl b/benchmark/runbench.jl
index 24c6221..9762708 100644
--- a/benchmark/runbench.jl
+++ b/benchmark/runbench.jl
@@ -47,9 +47,11 @@ function runbench()
     export_markdown(joinpath(@__DIR__, "results.md"), bench_results)
 end
 
+# For judging the difference between two commits, we don't need to divide by N. Keeping the
+# numbers larger like this helps the signal-to-noise ratio for comparing small changes.
 function judgebench(target::Union{String, BenchmarkConfig}, baseline::Union{String, BenchmarkConfig})
     bench_results = withenv("BENCH_NUM_ITERS"=>string(N)) do
-        judge("FixedPointDecimals", target, baseline; f=identity, postprocess=postprocess)
+        judge("FixedPointDecimals", target, baseline)
     end
 end
 function judgebench(baseline::Union{String, BenchmarkConfig})

From 6402fa2f3c1e1b30bac5e58218822bad77033514 Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Sun, 27 Jan 2019 10:30:47 -0500
Subject: [PATCH 12/15] put back postprocess; set N=1

---
 benchmark/runbench.jl | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/benchmark/runbench.jl b/benchmark/runbench.jl
index 9762708..dcb0df1 100644
--- a/benchmark/runbench.jl
+++ b/benchmark/runbench.jl
@@ -5,7 +5,7 @@ using Pkg
 Pkg.activate(@__DIR__)
 using PkgBenchmark, BenchmarkTools, Statistics
 
-const N = 1_000
+const N = 1 #_000
 
 import Base: -, /
 function -(a::BenchmarkTools.TrialEstimate, b::BenchmarkTools.TrialEstimate)
@@ -47,11 +47,9 @@ function runbench()
     export_markdown(joinpath(@__DIR__, "results.md"), bench_results)
 end
 
-# For judging the difference between two commits, we don't need to divide by N. Keeping the
-# numbers larger like this helps the signal-to-noise ratio for comparing small changes.
 function judgebench(target::Union{String, BenchmarkConfig}, baseline::Union{String, BenchmarkConfig})
     bench_results = withenv("BENCH_NUM_ITERS"=>string(N)) do
-        judge("FixedPointDecimals", target, baseline)
+        judge("FixedPointDecimals", target, baseline; f=identity, postprocess=postprocess)
     end
 end
 function judgebench(baseline::Union{String, BenchmarkConfig})

From 66a3172659f4e2f0ef56a6d211e4233e5709560f Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Sun, 27 Jan 2019 10:40:32 -0500
Subject: [PATCH 13/15] Keep N=1000; still no div for judge

---
 benchmark/runbench.jl | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/benchmark/runbench.jl b/benchmark/runbench.jl
index dcb0df1..95d5777 100644
--- a/benchmark/runbench.jl
+++ b/benchmark/runbench.jl
@@ -5,7 +5,7 @@ using Pkg
 Pkg.activate(@__DIR__)
 using PkgBenchmark, BenchmarkTools, Statistics
 
-const N = 1 #_000
+const N = 1_000
 
 import Base: -, /
 function -(a::BenchmarkTools.TrialEstimate, b::BenchmarkTools.TrialEstimate)
@@ -38,6 +38,22 @@ function postprocess(results::BenchmarkGroup)
     end
     results
 end
+function postprocess_no_div(results::BenchmarkGroup)
+    for (op, op_group) in results.data
+        op_results = op_group.data
+        for (type, type_group) in op_results
+            benchresults = type_group.data
+            if op == "identity"
+                # For :identity, bench and base are identical so we don't want to subtract.
+                op_results[type] = median(benchresults["bench"])
+            else
+                op_results[type] = median(benchresults["bench"]) - median(benchresults["base"])
+            end
+        end
+    end
+    results
+end
+
 
 function runbench()
     bench_results = withenv("BENCH_NUM_ITERS"=>string(N)) do
@@ -49,7 +65,7 @@ end
 
 function judgebench(target::Union{String, BenchmarkConfig}, baseline::Union{String, BenchmarkConfig})
     bench_results = withenv("BENCH_NUM_ITERS"=>string(N)) do
-        judge("FixedPointDecimals", target, baseline; f=identity, postprocess=postprocess)
+        judge("FixedPointDecimals", target, baseline; f=identity, postprocess=postprocess_no_div)
     end
 end
 function judgebench(baseline::Union{String, BenchmarkConfig})

From a58dbfd990c47d2da392af64ff801084a37a8b6e Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Sun, 27 Jan 2019 10:40:58 -0500
Subject: [PATCH 14/15] Remove non-FD benchmarks; they'll never change so
 shouldn't be a part of this

---
 benchmark/benchmarks.jl | 38 +++++++++++---------------------------
 1 file changed, 11 insertions(+), 27 deletions(-)

diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl
index 645c2eb..7c60957 100644
--- a/benchmark/benchmarks.jl
+++ b/benchmark/benchmarks.jl
@@ -20,33 +20,17 @@ using BenchmarkTools, Statistics
 const SUITE = BenchmarkGroup()
 const N = parse(Int, get(ENV, "BENCH_NUM_ITERS", "1000"))
 
-decimal_precision = 2
-
-# Express that data through the various types. Round it for integers.
-fd_FixedPointDecimal_types = [
-    FixedPointDecimals.FixedDecimal{Int32, decimal_precision},
-    FixedPointDecimals.FixedDecimal{Int64, decimal_precision},
-    FixedPointDecimals.FixedDecimal{Int128, decimal_precision},
+benchtypes = [
+    FixedPointDecimals.FixedDecimal{Int32,  2},
+    FixedPointDecimals.FixedDecimal{Int64,  2},
+    FixedPointDecimals.FixedDecimal{Int128, 2},
 ]
-inttypes = [Int32,Int64,Int128]
-floattypes = [Float32,Float64]
-#bigtypes = [BigInt, BigFloat]
-
-alltypes = (inttypes..., floattypes..., fd_FixedPointDecimal_types...,)
 
 identity1(a,_) = a
 allops = (*, /, +, ÷, identity1)
 
-# Category for the results output CSV
-category(::Type{<:Union{inttypes...}}) = "Int"
-category(::Type{<:Union{floattypes...}}) = "Float"
-#category(::Type{<:Union{bigtypes...}}) = "Big"
-category(::Type{<:FixedPointDecimals.FixedDecimal}) = "FixedDecimal"
-type(T::Type) = "$T"
-type(T::Type{<:Union{Int32, Int64}}) = "  $T"
-type(T::Type{Int128}) = " $T"
-type(::Type{FixedPointDecimals.FixedDecimal{T,f}}) where {T,f} = "FD{$T,$f}"
-type(::Type{FixedPointDecimals.FixedDecimal{T,f}}) where {T<:Union{Int32,Int64},f} = "FD{ $T,$f}"
+prettytype(::Type{FixedPointDecimals.FixedDecimal{T,f}}) where {T,f} = "FD{$T,$f}"
+prettytype(::Type{FixedPointDecimals.FixedDecimal{T,f}}) where {T<:Union{Int32,Int64},f} = "FD{ $T,$f}"
 opname(f) = string(Symbol(f))
 opname(f::typeof(identity1)) = "identity"
 
@@ -79,15 +63,15 @@ end
 # Define the benchmark structure
 for op in allops
     SUITE[opname(op)] = BenchmarkGroup()
-    for T in alltypes
-        SUITE[opname(op)][type(T)] = BenchmarkGroup(["base", "bench"])
+    for T in benchtypes
+        SUITE[opname(op)][prettytype(T)] = BenchmarkGroup(["base", "bench"])
     end
 end
 
 for op in allops
     println()
     println("$op")
-    for T in alltypes
+    for T in benchtypes
         print("$T ")
 
         initial_value = zero(T)
@@ -99,8 +83,8 @@ for op in allops
 
         # Run the benchmark
         outbase = Ref(initial_value)
-        SUITE[opname(op)][type(T)]["base"] = @benchmarkable $fbase($outbase) evals=1 setup=($outbase[]=$initial_value)
+        SUITE[opname(op)][prettytype(T)]["base"] = @benchmarkable $fbase($outbase) evals=1 setup=($outbase[]=$initial_value)
         outbench = Ref(initial_value)
-        SUITE[opname(op)][type(T)]["bench"] = @benchmarkable $fbench($outbench) evals=1 setup=($outbench[]=$initial_value)
+        SUITE[opname(op)][prettytype(T)]["bench"] = @benchmarkable $fbench($outbench) evals=1 setup=($outbench[]=$initial_value)
     end
 end

From ed2db1770db958a5248b0507ca1df6c52776f074 Mon Sep 17 00:00:00 2001
From: Nathan Daly <nhdaly@gmail.com>
Date: Mon, 28 Jan 2019 13:27:53 -0500
Subject: [PATCH 15/15] Added option to change postprocess function in
 judgebench for testing purposes

---
 benchmark/runbench.jl | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/benchmark/runbench.jl b/benchmark/runbench.jl
index 95d5777..45658c8 100644
--- a/benchmark/runbench.jl
+++ b/benchmark/runbench.jl
@@ -56,17 +56,27 @@ end
 
 
 function runbench()
+    rm(joinpath(@__DIR__, "tune.json"))  # Remove the existing tune.json file.
     bench_results = withenv("BENCH_NUM_ITERS"=>string(N)) do
         benchmarkpkg("FixedPointDecimals"; postprocess=postprocess)
     end
 
     export_markdown(joinpath(@__DIR__, "results.md"), bench_results)
+    return bench_results
 end
 
-function judgebench(target::Union{String, BenchmarkConfig}, baseline::Union{String, BenchmarkConfig})
+function judgebench(target::Union{String, BenchmarkConfig}, baseline::Union{String, BenchmarkConfig},
+            postprocess_fn=postprocess_no_div)
+    try rm(joinpath(@__DIR__, "tune.json")) catch end  # Remove the existing tune.json file.
     bench_results = withenv("BENCH_NUM_ITERS"=>string(N)) do
-        judge("FixedPointDecimals", target, baseline; f=identity, postprocess=postprocess_no_div)
+        if postprocess_fn != nothing
+            judge("FixedPointDecimals", target, baseline; f=identity, postprocess=postprocess_fn)
+        else
+            judge("FixedPointDecimals", target, baseline)
+        end
     end
+    export_markdown(joinpath(@__DIR__, "judge.md"), bench_results)
+    return bench_results
 end
 function judgebench(baseline::Union{String, BenchmarkConfig})
     judgebench(BenchmarkConfig(), baseline)