Mercurial > repos > public > sbplib_julia

comparison benchmark/benchmark_utils.jl @ 1858:4a9be96f2569 feature/documenter_logo

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merge default
author Jonatan Werpers <jonatan@werpers.com>
date Sun, 12 Jan 2025 21:18:44 +0100
parents 471a948cd2b2
children e1d64f4110bd
comparison
equal deleted inserted replaced
1857:ffde7dad9da5 1858:4a9be96f2569
1 import PkgBenchmark
2 import Markdown
3 import Mustache
4 import Dates
5
6 import Diffinitive
7
8 const diffinitive_root = splitpath(pathof(Diffinitive))[1:end-2] |> joinpath
9 const results_dir = mkpath(joinpath(diffinitive_root, "benchmark/results"))
10 const template_path = joinpath(diffinitive_root, "benchmark/result.tmpl")
11
12 """
13 mainmain(;rev=nothing, target=nothing, baseline=nothing , kwargs...)
14
15 Calls `run_benchmark(args...; kwargs...)` and writes the results as an HTML
16 file in `benchmark/results`.
17
18 * If `rev` is set, the benchmarks are run for the given mercurial revision.
19 * If only `baseline` is set, the current working directory is compared with
20 the revision given in `baseline`.
21 * If both `target` and `baseline` is set those revision are compared.
22
23 For control over what happens to the benchmark result datastructure see the
24 different methods of [`run_benchmark`](@ref)
25 """
26 function main(;rev=nothing, target=nothing, baseline=nothing, name=nothing, kwargs...)
27 if !isnothing(rev)
28 r = run_benchmark(rev; kwargs...)
29 elseif !isnothing(baseline)
30 if isnothing(target)
31 r = compare_benchmarks(baseline; kwargs...)
32 else
33 r = compare_benchmarks(target, baseline; kwargs...)
34 end
35 else
36 # Neither rev, or baseline were set => Run on current working directory.
37 r = run_benchmark(;kwargs...)
38 end
39
40 file_path = write_result_html(r; name)
41 open_in_default_browser(file_path)
42 end
43
44
45 """
46 run_benchmark()
47
48 Run the benchmark suite for the current working directory and return a
49 `PkgBenchmark.BenchmarkResult`
50 """
51 function run_benchmark(;kwargs...)
52 r = PkgBenchmark.benchmarkpkg(Diffinitive; kwargs...)
53
54 rev = hg_rev() # Should be changed to hg_id() when the html can handle it.
55
56 return add_rev_info(r, rev)
57 end
58
59 """
60 run_benchmark(rev)
61
62 Updates the repository to the given revison and runs the benchmark suite. When
63 done, reverts the repository to the original state. `rev` can be any
64 identifier compatible with `hg update`.
65
66 Returns a `PkgBenchmark.BenchmarkResult`
67 """
68 function run_benchmark(rev; kwargs...)
69 return hg_at_revision(rev) do
70 run_benchmark(; kwargs...)
71 end
72 end
73
74 """
75 compare_benchmarks(target, baseline, f=minimum; judgekwargs=Dict())
76
77 Runs the benchmark at revisions `target` and `baseline` and compares them
78 using `PkgBenchmark.judge`. `f` is the function used to compare. `judgekwargs`
79 are keyword arguments passed to `judge`.
80
81 `target` and `baseline` can be any identifier compatible with `hg update`.
82
83 Returns a `PkgBenchmark.BenchmarkJudgement`
84 """
85 function compare_benchmarks(target, baseline, f=minimum; judgekwargs=Dict(), kwargs...)
86 t = run_benchmark(target; kwargs...)
87 b = run_benchmark(baseline; kwargs...)
88
89 return PkgBenchmark.judge(t,b,f; judgekwargs...)
90 end
91
92 """
93 compare_benchmarks(baseline, ...)
94
95 Compare the results at the current working directory with the revision
96 specified in `baseline`.
97
98 Accepts the same arguments as the two revision version.
99 """
100 function compare_benchmark(baseline, f=minimum; judgekwargs=Dict(), kwargs...)
101 t = run_benchmark(;kwargs...)
102 b = run_benchmark(baseline; kwargs...)
103
104 return PkgBenchmark.judge(t,b,f; judgekwargs...)
105 end
106
107
108 function add_rev_info(benchmarkresult, rev)
109 if endswith(rev,"+")
110 revstr = "+$rev" # Workaround for the bad presentation of BenchmarkResults.
111 else
112 revstr = rev
113 end
114
115 return PkgBenchmark.BenchmarkResults(
116 benchmarkresult.name,
117 revstr,
118 benchmarkresult.benchmarkgroup,
119 benchmarkresult.date,
120 benchmarkresult.julia_commit,
121 benchmarkresult.vinfo,
122 benchmarkresult.benchmarkconfig,
123 )
124 end
125
126
127 function write_result_html(io, r)
128 iobuffer = IOBuffer()
129 PkgBenchmark.export_markdown(iobuffer, r)
130
131 parsed_md = Markdown.parse(String(take!(iobuffer)))
132 content = Markdown.html(parsed_md)
133
134 template = Mustache.load(template_path)
135
136 dt = Dates.format(PkgBenchmark.date(r), "yyyy-mm-dd HH:MM:SS")
137 Mustache.render(io, template, Dict("title"=>dt, "content"=>content))
138 end
139
140 function write_result_html(r; name=nothing)
141 dt = Dates.format(PkgBenchmark.date(r), "yyyy-mm-dd HHMMSS")
142
143 if isnothing(name)
144 file_path = joinpath(results_dir, dt*".html")
145 else
146 file_path = joinpath(results_dir, dt*" "*name*".html")
147 end
148
149 open(file_path, "w") do io
150 write_result_html(io, r)
151 end
152
153 return file_path
154 end
155
156
157 PkgBenchmark.date(j::PkgBenchmark.BenchmarkJudgement) = PkgBenchmark.date(PkgBenchmark.target_result(j))
158
159
160 function hg_id()
161 cmd = Cmd(`hg id`, dir=diffinitive_root)
162 return readchomp(addenv(cmd, "HGPLAIN"=>""))
163 end
164
165 function hg_rev()
166 cmd = Cmd(`hg id -i`, dir=diffinitive_root)
167 return readchomp(addenv(cmd, "HGPLAIN"=>""))
168 end
169
170 function hg_update(rev)
171 cmd = Cmd(`hg update --check -r $rev`, dir=diffinitive_root)
172 run(addenv(cmd, "HGPLAIN"=>""))
173
174 return nothing
175 end
176
177 """
178 hg_commit(msg; secret=false)
179
180 Make a hg commit with the provided message. If `secret` is true the commit is
181 in the secret phase stopping it from being pushed.
182 """
183 function hg_commit(msg; secret=false)
184 if secret
185 cmd = Cmd(`hg commit --verbose --secret --message $msg`, dir=diffinitive_root)
186 else
187 cmd = Cmd(`hg commit --verbose --message $msg`, dir=diffinitive_root)
188 end
189
190 out = readchomp(addenv(cmd, "HGPLAIN"=>""))
191
192 return only(match(r"committed changeset \d+:([0-9a-z]+)", out))
193 end
194
195 """
196 hg_strip(rev; keep=false)
197
198 Strips the given commit from the repo. If `keep` is true, the changes of the
199 commit are kept in the working directory.
200 """
201 function hg_strip(rev; keep=false)
202 if keep
203 cmd = Cmd(`hg --config extensions.strip= strip --keep -r $rev`, dir=diffinitive_root)
204 else
205 cmd = Cmd(`hg --config extensions.strip= strip -r $rev`, dir=diffinitive_root)
206 end
207
208 run(addenv(cmd, "HGPLAIN"=>""))
209
210 return nothing
211 end
212
213 """
214 hg_is_dirty()
215
216 Return true if the repositopry has uncommited changes.
217 """
218 function hg_is_dirty()
219 cmd = Cmd(`hg identify --id`, dir=diffinitive_root)
220 out = readchomp(addenv(cmd, "HGPLAIN"=>""))
221
222 return endswith(out, "+")
223 end
224
225 """
226 hg_at_revision(f, rev)
227
228 Update the repository to the given revision and run the function `f`. After
229 `f` is run the working directory is restored. If there are uncommited changes
230 a temporary commit will be used to save the state of the working directory.
231 """
232 function hg_at_revision(f, rev)
233 if hg_is_dirty()
234 hg_with_temporary_commit() do
235 return _hg_at_revision(f, rev)
236 end
237 else
238 return _hg_at_revision(f, rev)
239 end
240 end
241
242 function _hg_at_revision(f, rev)
243 @assert !hg_is_dirty()
244
245 origin_rev = hg_rev()
246
247 hg_update(rev)
248 try
249 return f()
250 finally
251 hg_update(origin_rev)
252 end
253 end
254
255 """
256 hg_with_temporary_commit(f)
257
258 Run the function `f` after making a temporary commit with the current working
259 directory. After `f` has finished the working directory is restored to its
260 original state and the temporary commit stripped.
261 """
262 function hg_with_temporary_commit(f)
263 @assert hg_is_dirty()
264
265 origin_rev = hg_commit("[Automatic commit by julia]",secret=true)
266
267 try
268 return f()
269 finally
270 hg_update(origin_rev)
271 hg_strip(origin_rev; keep=true)
272 end
273 end
274
275
276 # From Pluto.jl/src/webserver/WebServer.jl (2023-01-24)
277 function open_in_default_browser(url::AbstractString)::Bool
278 try
279 if Sys.isapple()
280 Base.run(`open $url`)
281 true
282 elseif Sys.iswindows() || detectwsl()
283 Base.run(`powershell.exe Start "'$url'"`)
284 true
285 elseif Sys.islinux()
286 Base.run(`xdg-open $url`)
287 true
288 else
289 false
290 end
291 catch ex
292 false
293 end
294 end
295
296
297 main
298
299 # TODO: Better logging of what is happening
300 # TODO: Improve the workflow? How?
301
302 # TODO: Clean up the HTML output?
303 # TODO: Make the codeblocks in the table look nicer
304 # TODO: Change width of tables and code blocks so everything is visible
305 # TODO: Fix the commit id, it chops off all the important info
306 # TODO: Make title less verbose
307 # TBD: Do we have to replace export_markdown? Could use a template instead.
308
309 # Should be able to run the current benchmark script at a different revision.
310 # Should have a way to filter the benchmark suite
311
312 # TBD: What parts are PkgBenchmark contributing? Can it be stripped out?
313
314
315 ## Catching the exit code and errors from a command can be done with code similar to
316 # proc = open(cmd)
317 # if success(proc)
318
319 # else
320
321 # end