Mercurial > repos > public > sbplib_julia
view benchmark/benchmark_utils.jl @ 2015:5c2448d6a201 feature/grids/geometry_functions tip
Structure tests a bit more
author | Jonatan Werpers <jonatan@werpers.com> |
---|---|
date | Fri, 09 May 2025 15:57:38 +0200 |
parents | 9d708f3300d5 |
children |
line wrap: on
line source
import PkgBenchmark import Markdown import Mustache import Dates import Diffinitive const diffinitive_root = splitpath(pathof(Diffinitive))[1:end-2] |> joinpath const results_dir = mkpath(joinpath(diffinitive_root, "benchmark/results")) const template_path = joinpath(diffinitive_root, "benchmark/result.tmpl") """ main(;rev=nothing, target=nothing, baseline=nothing , kwargs...) Calls `run_benchmark(args...; kwargs...)` and writes the results as an HTML file in `benchmark/results`. * If `rev` is set, the benchmarks are run for the given mercurial revision. * If only `baseline` is set, the current working directory is compared with the revision given in `baseline`. * If both `target` and `baseline` is set those revision are compared. For control over what happens to the benchmark result datastructure see the different methods of [`run_benchmark`](@ref) """ function main(;rev=nothing, target=nothing, baseline=nothing, name=nothing, kwargs...) if !isnothing(rev) r = run_benchmark(rev; kwargs...) elseif !isnothing(baseline) if isnothing(target) r = compare_benchmarks(baseline; kwargs...) else r = compare_benchmarks(target, baseline; kwargs...) end else # Neither rev, or baseline were set => Run on current working directory. r = run_benchmark(;kwargs...) end file_path = write_result_html(r; name) open_in_default_browser(file_path) end """ run_benchmark() Run the benchmark suite for the current working directory and return a `PkgBenchmark.BenchmarkResult` """ function run_benchmark(;kwargs...) r = PkgBenchmark.benchmarkpkg(Diffinitive; kwargs...) rev = hg_rev() # Should be changed to hg_id() when the html can handle it. return add_rev_info(r, rev) end """ run_benchmark(rev) Updates the repository to the given revison and runs the benchmark suite. When done, reverts the repository to the original state. `rev` can be any identifier compatible with `hg update`. Returns a `PkgBenchmark.BenchmarkResult` """ function run_benchmark(rev; kwargs...) return hg_at_revision(rev) do run_benchmark(; kwargs...) end end """ compare_benchmarks(target, baseline, f=minimum; judgekwargs=Dict()) Runs the benchmark at revisions `target` and `baseline` and compares them using `PkgBenchmark.judge`. `f` is the function used to compare. `judgekwargs` are keyword arguments passed to `judge`. `target` and `baseline` can be any identifier compatible with `hg update`. Returns a `PkgBenchmark.BenchmarkJudgement` """ function compare_benchmarks(target, baseline; f=minimum, judgekwargs=Dict(), kwargs...) t = run_benchmark(target; kwargs...) b = run_benchmark(baseline; kwargs...) return PkgBenchmark.judge(t,b,f; judgekwargs...) end """ compare_benchmarks(baseline, ...) Compare the results at the current working directory with the revision specified in `baseline`. Accepts the same arguments as the two revision version. """ function compare_benchmarks(baseline; f=minimum, judgekwargs=Dict(), kwargs...) t = run_benchmark(;kwargs...) b = run_benchmark(baseline; kwargs...) return PkgBenchmark.judge(t,b,f; judgekwargs...) end function add_rev_info(benchmarkresult, rev) if endswith(rev,"+") revstr = "+$rev" # Workaround for the bad presentation of BenchmarkResults. else revstr = rev end return PkgBenchmark.BenchmarkResults( benchmarkresult.name, revstr, benchmarkresult.benchmarkgroup, benchmarkresult.date, benchmarkresult.julia_commit, benchmarkresult.vinfo, benchmarkresult.benchmarkconfig, ) end function write_result_html(io, r) iobuffer = IOBuffer() PkgBenchmark.export_markdown(iobuffer, r) parsed_md = Markdown.parse(String(take!(iobuffer))) content = Markdown.html(parsed_md) template = Mustache.load(template_path) dt = Dates.format(PkgBenchmark.date(r), "yyyy-mm-dd HH:MM:SS") Mustache.render(io, template, Dict("title"=>dt, "content"=>content)) end function write_result_html(r; name=nothing) dt = Dates.format(PkgBenchmark.date(r), "yyyy-mm-dd HHMMSS") if isnothing(name) file_path = joinpath(results_dir, dt*".html") else file_path = joinpath(results_dir, dt*" "*name*".html") end open(file_path, "w") do io write_result_html(io, r) end return file_path end PkgBenchmark.date(j::PkgBenchmark.BenchmarkJudgement) = PkgBenchmark.date(PkgBenchmark.target_result(j)) function hg_id() cmd = Cmd(`hg id`, dir=diffinitive_root) return readchomp(addenv(cmd, "HGPLAIN"=>"")) end function hg_rev() cmd = Cmd(`hg id -i`, dir=diffinitive_root) return readchomp(addenv(cmd, "HGPLAIN"=>"")) end function hg_update(rev) cmd = Cmd(`hg update --check -r $rev`, dir=diffinitive_root) run(addenv(cmd, "HGPLAIN"=>"")) return nothing end """ hg_commit(msg; secret=false) Make a hg commit with the provided message. If `secret` is true the commit is in the secret phase stopping it from being pushed. """ function hg_commit(msg; secret=false) if secret cmd = Cmd(`hg commit --verbose --secret --message $msg`, dir=diffinitive_root) else cmd = Cmd(`hg commit --verbose --message $msg`, dir=diffinitive_root) end out = readchomp(addenv(cmd, "HGPLAIN"=>"")) return only(match(r"committed changeset \d+:([0-9a-z]+)", out)) end """ hg_strip(rev; keep=false) Strips the given commit from the repo. If `keep` is true, the changes of the commit are kept in the working directory. """ function hg_strip(rev; keep=false) if keep cmd = Cmd(`hg --config extensions.strip= strip --keep -r $rev`, dir=diffinitive_root) else cmd = Cmd(`hg --config extensions.strip= strip -r $rev`, dir=diffinitive_root) end run(addenv(cmd, "HGPLAIN"=>"")) return nothing end """ hg_is_dirty() Return true if the repositopry has uncommited changes. """ function hg_is_dirty() cmd = Cmd(`hg identify --id`, dir=diffinitive_root) out = readchomp(addenv(cmd, "HGPLAIN"=>"")) return endswith(out, "+") end """ hg_at_revision(f, rev) Update the repository to the given revision and run the function `f`. After `f` is run the working directory is restored. If there are uncommited changes a temporary commit will be used to save the state of the working directory. """ function hg_at_revision(f, rev) if hg_is_dirty() hg_with_temporary_commit() do return _hg_at_revision(f, rev) end else return _hg_at_revision(f, rev) end end function _hg_at_revision(f, rev) @assert !hg_is_dirty() origin_rev = hg_rev() hg_update(rev) try return f() finally hg_update(origin_rev) end end """ hg_with_temporary_commit(f) Run the function `f` after making a temporary commit with the current working directory. After `f` has finished the working directory is restored to its original state and the temporary commit stripped. """ function hg_with_temporary_commit(f) @assert hg_is_dirty() origin_rev = hg_commit("[Automatic commit by julia]",secret=true) try return f() finally hg_update(origin_rev) hg_strip(origin_rev; keep=true) end end # From Pluto.jl/src/webserver/WebServer.jl (2023-01-24) function open_in_default_browser(url::AbstractString)::Bool try if Sys.isapple() Base.run(`open $url`) true elseif Sys.iswindows() || detectwsl() Base.run(`powershell.exe Start "'$url'"`) true elseif Sys.islinux() Base.run(`xdg-open $url`) true else false end catch ex false end end main # TODO: Better logging of what is happening # TODO: Improve the workflow? How? # TODO: Clean up the HTML output? # TODO: Make the codeblocks in the table look nicer # TODO: Change width of tables and code blocks so everything is visible # TODO: Fix the commit id, it chops off all the important info # TODO: Make title less verbose # TBD: Do we have to replace export_markdown? Could use a template instead. # Should be able to run the current benchmark script at a different revision. # Should have a way to filter the benchmark suite # TBD: What parts are PkgBenchmark contributing? Can it be stripped out? ## Catching the exit code and errors from a command can be done with code similar to # proc = open(cmd) # if success(proc) # else # end