changeset 1320:6ae2ec4cef5a tooling/benchmarks

Rename functions for comparing benchmarks. Refactor their code. Allow comparing the current working directory
author Jonatan Werpers <jonatan@werpers.com>
date Fri, 28 Apr 2023 08:53:39 +0200
parents cb4d57ce2ed1
children 42738616422e
files benchmark/benchmark_utils.jl benchmark/make.jl
diffstat 2 files changed, 61 insertions(+), 21 deletions(-) [+]
line wrap: on
line diff
diff -r cb4d57ce2ed1 -r 6ae2ec4cef5a benchmark/benchmark_utils.jl
--- a/benchmark/benchmark_utils.jl	Fri Apr 28 08:37:04 2023 +0200
+++ b/benchmark/benchmark_utils.jl	Fri Apr 28 08:53:39 2023 +0200
@@ -10,13 +10,33 @@
 const template_path = joinpath(sbplib_root, "benchmark/result.tmpl")
 
 """
- main(args...; kwargs...)
+    mainmain(;rev=nothing, target=nothing, baseline=nothing , kwargs...)
+
+Calls `run_benchmark(args...; kwargs...)` and writes the results as an HTML
+file in `benchmark/results`.
+
+ * If `rev` is set, the benchmarks are run for the given mercurial revision.
+ * If only `baseline` is set, the current working directory is compared with
+   the revision given in `baseline`.
+ * If  both `target` and `baseline` is set those revision are compared.
 
-Calls `run_benchmark(args...; kwargs...)` and writes the results as an HTML file in `benchmark/results`.
-See [`run_benchmark`](@ref) for possible arguments.
+For control over what happens to the benchmark result datastructure see the
+different methods of [`run_benchmark`](@ref)
 """
-function main(args...; kwargs...)
-    r = run_benchmark(args...; kwargs...)
+function main(;rev=nothing, target=nothing, baseline=nothing , kwargs...)
+    if !isnothing(rev)
+        r = run_benchmark(rev; kwargs...)
+    elseif !isnothing(baseline)
+        if isnothing(target)
+            r = compare_benchmarks(baseline; kwargs...)
+        else
+            r = compare_benchmarks(target, baseline; kwargs...)
+        end
+    else
+        # Neither rev, or baseline were set => Run on current working directory.
+        r = run_benchmark(;kwargs...)
+    end
+
     file_path = write_result_html(r)
     open_in_default_browser(file_path)
 end
@@ -25,7 +45,8 @@
 """
     run_benchmark()
 
-Runs the benchmark suite for the current working directory and returns a `PkgBenchmark.BenchmarkResult`
+Run the benchmark suite for the current working directory and return a
+`PkgBenchmark.BenchmarkResult`
 """
 function run_benchmark(;kwargs...)
     r = PkgBenchmark.benchmarkpkg(Sbplib; kwargs...)
@@ -46,12 +67,12 @@
 """
 function run_benchmark(rev; kwargs...)
     return hg_at_revision(rev) do
-        run_benchmark(;kwargs...)
+        run_benchmark(; kwargs...)
     end
 end
 
 """
-    run_benchmark(target, baseline, f=minimum; judgekwargs=Dict())
+    compare_benchmarks(target, baseline, f=minimum; judgekwargs=Dict())
 
 Runs the benchmark at revisions `target` and `baseline` and compares them
 using `PkgBenchmark.judge`. `f` is the function used to compare. `judgekwargs`
@@ -61,13 +82,24 @@
 
 Returns a `PkgBenchmark.BenchmarkJudgement`
 """
-function run_benchmark(target, baseline, f=minimum; judgekwargs=Dict(), kwargs...)
-    rev_before = hg_rev()
-    hg_update(target)
+function compare_benchmarks(target, baseline, f=minimum; judgekwargs=Dict(), kwargs...)
+    t = run_benchmark(target; kwargs...)
+    b = run_benchmark(baseline; kwargs...)
+
+    return PkgBenchmark.judge(t,b,f; judgekwargs...)
+end
+
+"""
+    compare_benchmarks(baseline, ...)
+
+Compare the results at the current working directory with the revision
+specified in `baseline`.
+
+Accepts the same arguments as the two revision version.
+"""
+function compare_benchmark(baseline, f=minimum; judgekwargs=Dict(), kwargs...)
     t = run_benchmark(;kwargs...)
-    hg_update(baseline)
-    b = run_benchmark(;kwargs...)
-    hg_update(rev_before)
+    b = run_benchmark(baseline; kwargs...)
 
     return PkgBenchmark.judge(t,b,f; judgekwargs...)
 end
diff -r cb4d57ce2ed1 -r 6ae2ec4cef5a benchmark/make.jl
--- a/benchmark/make.jl	Fri Apr 28 08:37:04 2023 +0200
+++ b/benchmark/make.jl	Fri Apr 28 08:53:39 2023 +0200
@@ -1,13 +1,21 @@
+rev = nothing
+baseline = nothing
+target = nothing
+
 if "--rev" ∈ ARGS
     i = findlast(==("--rev"), ARGS)
-    args = parse(Int,ARGS[i+1])
-elseif ("--target","--baseline") ∈ ARGS
+    rev = ARGS[i+1]
+end
+
+if "--target" ∈ ARGS
     i = findlast(==("--target"), ARGS)
-    j = findlast(==("--baseline"), ARGS)
-    args = (ARGS[i+1],ARGS[j+1])
-else
-    args = ()
+    target = ARGS[i+1]
+end
+
+if "--baseline" ∈ ARGS
+    i = findlast(==("--baseline"), ARGS)
+    baseline = ARGS[i+1]
 end
 
 include("benchmark_utils.jl")
-main(args...)
\ No newline at end of file
+main(;rev, target, baseline)