changeset 1316:8e5d4ddf7afd tooling/benchmarks

Merge
author Jonatan Werpers <jonatan@werpers.com>
date Thu, 27 Apr 2023 16:12:13 +0200
parents 0551fd71682f (diff) 7610a61eaac1 (current diff)
children 7110851b9c87
files
diffstat 4 files changed, 146 insertions(+), 22 deletions(-) [+]
line wrap: on
line diff
--- a/Makefile	Thu Mar 23 06:57:14 2023 +0100
+++ b/Makefile	Thu Apr 27 16:12:13 2023 +0200
@@ -44,11 +44,14 @@
 	rm -rf docs/build
 	rm -rf docs/build-local	
 
-benchmark: benchmark/run-benchmark
+benchmark:
+	$(JULIA) --project=benchmark benchmark/make.jl
 
-benchmarkrev: benchmark/run-benchmark-rev
+benchmarkrev:
+	$(JULIA) --project=benchmark benchmark/make.jl --rev $(REV)
 
-benchmarkcmp: benchmark/run-benchmark-cmp
+benchmarkcmp:
+	$(JULIA) --project=benchmark benchmark/make.jl --cmp $(TARGET) $(BASELINE)
 
 cleanbenchmark:
 	rm -rf benchmark/results
@@ -67,17 +70,3 @@
 
 docs/build-local: $(DOCS_DEPENDENCIES)
 	$(JULIA) --project=docs docs/make.jl --build-dir build-local
-
-benchmark/run-benchmark:
-	$(JULIA) --project=benchmark benchmark/make.jl
-
-benchmark/run-benchmark-rev:
-	$(JULIA) --project=benchmark benchmark/make.jl --rev $(REV)
-
-benchmark/run-benchmark-cmp:
-	$(JULIA) --project=benchmark benchmark/make.jl --cmp $(TARGET) $(BASELINE)
-
-
-.PHONY: temp
-temp:
-	@echo $(SRC_FILES_AND_DIRS)
--- a/README.md	Thu Mar 23 06:57:14 2023 +0100
+++ b/README.md	Thu Apr 27 16:12:13 2023 +0200
@@ -72,7 +72,7 @@
 ```shell
 make docs          # generates files suitable for webserver deployment, i.e with `prettyurls=true`
 make localdocs     # generates files suitable for local viewing in a web browser, i.e `prettyurls=false`
-make opendocs      # view documentation locally
+make opendocs      # build and view documentation locally
 make cleandocs     # cleans up generated files
 ```
 
--- a/benchmark/benchmark_utils.jl	Thu Mar 23 06:57:14 2023 +0100
+++ b/benchmark/benchmark_utils.jl	Thu Apr 27 16:12:13 2023 +0200
@@ -30,7 +30,7 @@
 function run_benchmark(;kwargs...)
     r = PkgBenchmark.benchmarkpkg(Sbplib; kwargs...)
 
-    rev = hg_id()
+    rev = hg_rev() # Should be changed to hg_id() when the html can handle it.
 
     return add_rev_info(r, rev)
 end
@@ -38,8 +38,9 @@
 """
     run_benchmark(rev)
 
-Updates the repository to the given revison and runs the benchmark suite. When done, updates the repository to the origianl state.
-`rev` can be any identifier compatible with `hg update`.
+Updates the repository to the given revison and runs the benchmark suite. When
+done, reverts the repository to the original state. `rev` can be any
+identifier compatible with `hg update`.
 
 Returns a `PkgBenchmark.BenchmarkResult`
 """
@@ -75,9 +76,15 @@
 
 
 function add_rev_info(benchmarkresult, rev)
+    if endswith(rev,"+")
+        revstr = "+$rev" # Workaround for the bad presentation of BenchmarkResults.
+    else
+        revstr = rev
+    end
+
     return PkgBenchmark.BenchmarkResults(
         benchmarkresult.name,
-        rev,
+        revstr,
         benchmarkresult.benchmarkgroup,
         benchmarkresult.date,
         benchmarkresult.julia_commit,
@@ -128,6 +135,106 @@
 function hg_update(rev)
     cmd = Cmd(`hg update --check -r $rev`, dir=sbplib_root)
     run(addenv(cmd, "HGPLAIN"=>""))
+
+    return nothing
+end
+
+"""
+    hg_commit(msg; secret=false)
+
+Make a hg commit with the provided message. If `secret` is true the commit is
+in the secret phase stopping it from being pushed.
+"""
+function hg_commit(msg; secret=false)
+    if secret
+        cmd = Cmd(`hg commit --verbose --secret --message $msg`, dir=sbplib_root)
+    else
+        cmd = Cmd(`hg commit --verbose          --message $msg`, dir=sbplib_root)
+    end
+
+    out = readchomp(addenv(cmd, "HGPLAIN"=>""))
+
+    return only(match(r"committed changeset \d+:([0-9a-z]+)", out))
+end
+
+"""
+    hg_strip(rev; keep=false)
+
+Strips the given commit from the repo. If `keep` is true, the changes of the
+commit are kept in the working directory.
+"""
+function hg_strip(rev; keep=false)
+    if keep
+        cmd = Cmd(`hg --config extensions.strip= strip --keep -r $rev`, dir=sbplib_root)
+    else
+        cmd = Cmd(`hg --config extensions.strip= strip        -r $rev`, dir=sbplib_root)
+    end
+
+    run(addenv(cmd, "HGPLAIN"=>""))
+
+    return nothing
+end
+
+"""
+    hg_is_dirty()
+
+Return true if the repositopry has uncommited changes.
+"""
+function hg_is_dirty()
+    cmd = Cmd(`hg identify --id`, dir=sbplib_root)
+    out = readchomp(addenv(cmd, "HGPLAIN"=>""))
+
+    return endswith(out, "+")
+end
+
+"""
+    hg_at_revision(f, rev)
+
+Update the repository to the given revision and run the function `f`. After
+`f` is run the working directory is restored. If there are uncommited changes
+a temporary commit will be used to save the state of the working directory.
+"""
+function hg_at_revision(f, rev)
+    if hg_is_dirty()
+        hg_with_temporary_commit() do
+            return _hg_at_revision(f, rev)
+        end
+    else
+        return _hg_at_revision(f, rev)
+    end
+end
+
+function _hg_at_revision(f, rev)
+    @assert !hg_is_dirty()
+
+    origin_rev = hg_rev()
+
+    hg_update(rev)
+    try
+        return f()
+    finally
+        hg_update(origin_rev)
+    end
+end
+
+"""
+    hg_with_temporary_commit(f)
+
+Run the function `f` after making a temporary commit with the current working
+directory. After `f` has finished the working directory is restored to its
+original state and the temporary commit stripped.
+"""
+function hg_with_temporary_commit(f)
+    @assert hg_is_dirty()
+
+    origin_rev = hg_commit("[Automatic commit by julia]",secret=true)
+
+    try
+        return f()
+    finally
+        hg_update(origin_rev)
+        hg_strip(origin_rev; keep=true)
+    end
 end
 
 
@@ -166,4 +273,17 @@
 
 
 # TBD: How to compare against current working directory? Possible to create a temporary commit?
+    # Make a secret temporary commit
+    # run benchmarkresult
+    # uncommit the temporary commit
+    # verbose flag on commit will print the revision id of the new commit.
 # TBD: What parts are PkgBenchmark contributing? Can it be stripped out?
+
+
+## Catching the exit code and errors from a command can be done with code similar to
+    # proc = open(cmd)
+    # if success(proc)
+
+    # else
+
+    # end
--- a/benchmark/benchmarks.jl	Thu Mar 23 06:57:14 2023 +0100
+++ b/benchmark/benchmarks.jl	Thu Apr 27 16:12:13 2023 +0200
@@ -3,6 +3,7 @@
 using Sbplib.Grids
 using Sbplib.SbpOperators
 using Sbplib.RegionIndices
+using Sbplib.LazyTensors
 
 const SUITE = BenchmarkGroup()
 
@@ -162,4 +163,18 @@
 
 #TODO: Reorg with dimension as first level? To reduce operator creation?
 
+
+
+SUITE["lazy_tensors"] = BenchmarkGroup()
+
+SUITE["lazy_tensors"]["compositions"] = BenchmarkGroup()
+s = ScalingTensor(1.,(10,))
+u = rand(10)
+v = similar(u)
+s3 = s∘s∘s
+s4 = s∘s∘s∘s
+SUITE["lazy_tensors"]["compositions"]["s∘s∘s"]   = @benchmarkable $v .= $s3*$u
+SUITE["lazy_tensors"]["compositions"]["s∘s∘s∘s"] = @benchmarkable $v .= $s4*$u
+
+
 SUITE