From 119741be086852d8e652d09c820c025e57ce22a4 Mon Sep 17 00:00:00 2001 From: Hossein Pourbozorg Date: Wed, 16 Aug 2023 07:39:20 +0330 Subject: [PATCH] add `BenchmarkCI` --- .github/workflows/Benchmark.yml | 25 ++++++++++++++++++ .github/workflows/CI.yml | 1 - .gitignore | 3 +++ benchmark/Project.toml | 3 +++ benchmark/benchmarks.jl | 37 +++++++++++++++++++++++++++ test/Project.toml | 1 - test/benchmark_tests.jl | 45 --------------------------------- test/runtests.jl | 5 ---- 8 files changed, 68 insertions(+), 52 deletions(-) create mode 100644 .github/workflows/Benchmark.yml create mode 100644 benchmark/Project.toml create mode 100644 benchmark/benchmarks.jl delete mode 100644 test/benchmark_tests.jl diff --git a/.github/workflows/Benchmark.yml b/.github/workflows/Benchmark.yml new file mode 100644 index 00000000..71830519 --- /dev/null +++ b/.github/workflows/Benchmark.yml @@ -0,0 +1,25 @@ +name: Run benchmarks +on: pull_request + +jobs: + Benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: julia-actions/setup-julia@v1 + with: + version: '1' + arch: x64 + show-versioninfo: true + - uses: julia-actions/cache@v1 + - uses: julia-actions/julia-buildpkg@v1 + - name: Install dependencies + run: julia --color=yes -e 'using Pkg; Pkg.add(["PkgBenchmark", "BenchmarkCI"])' + - name: Run benchmarks + run: julia --color=yes -e 'using BenchmarkCI; BenchmarkCI.judge()' + - name: Print judgement + run: julia --color=yes -e 'using BenchmarkCI; BenchmarkCI.displayjudgement()' + - name: Post results + run: julia --color=yes -e 'using BenchmarkCI; BenchmarkCI.postjudge()' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index b36370d0..1bd5f908 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -16,7 +16,6 @@ jobs: - RNODE - FFJORD - Planar - - Benchmark - Quality - Instability version: diff --git a/.gitignore b/.gitignore index 204895bf..7e7dc9a0 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ Manifest.toml LocalPreferences.toml .vscode + +.benchmarkci +benchmark/*.json diff --git a/benchmark/Project.toml b/benchmark/Project.toml new file mode 100644 index 00000000..16acc497 --- /dev/null +++ b/benchmark/Project.toml @@ -0,0 +1,3 @@ +[deps] +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl new file mode 100644 index 00000000..8a28115c --- /dev/null +++ b/benchmark/benchmarks.jl @@ -0,0 +1,37 @@ +using ContinuousNormalizingFlows, BenchmarkTools, PkgBenchmark + +SUITE = BenchmarkGroup() + +SUITE["main"] = BenchmarkGroup(["package", "simple"]) +SUITE["main"]["0-order"] = BenchmarkGroup(["direct"]) +SUITE["main"]["1-order"] = BenchmarkGroup(["gradient"]) + +BenchmarkTools.DEFAULT_PARAMETERS.samples = 2^13 +BenchmarkTools.DEFAULT_PARAMETERS.seconds = convert(Float64, 2 * 60) +BenchmarkTools.DEFAULT_PARAMETERS.evals = 1 +BenchmarkTools.DEFAULT_PARAMETERS.overhead = 0.0 +BenchmarkTools.DEFAULT_PARAMETERS.gctrial = true +BenchmarkTools.DEFAULT_PARAMETERS.gcsample = false +BenchmarkTools.DEFAULT_PARAMETERS.time_tolerance = convert(Float64, eps(Float16)) +BenchmarkTools.DEFAULT_PARAMETERS.memory_tolerance = convert(Float64, eps(Float16)) + +nvars = 8 +n = 128 +rng = Random.default_rng() +r = rand(Float32, nvars, n) + +nn = FluxCompatLayer(Flux.f32(Flux.Dense(nvars => nvars, tanh))) +icnf = construct(RNODE, nn, nvars; compute_mode = ZygoteMatrixMode) + +ps, st = Lux.setup(rng, icnf) +diff_loss_train(x) = loss(icnf, TrainMode(), r, x, st) +diff_loss_test(x) = loss(icnf, TestMode(), r, x, st) +grad_diff_loss_train() = Zygote.gradient(diff_loss_train, ps) +grad_diff_loss_test() = Zygote.gradient(diff_loss_test, ps) +t_loss_train() = loss(icnf, TrainMode(), r, ps, st) +t_loss_test() = loss(icnf, TestMode(), r, ps, st) + +SUITE["main"]["0-order"]["train"] = @benchmarkable $t_loss_train() +SUITE["main"]["0-order"]["test"] = @benchmarkable $t_loss_test() +SUITE["main"]["1-order"]["train"] = @benchmarkable $grad_diff_loss_train() +SUITE["main"]["1-order"]["test"] = @benchmarkable $grad_diff_loss_test() diff --git a/test/Project.toml b/test/Project.toml index ba991c35..5e521ad5 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -2,7 +2,6 @@ ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" AbstractDifferentiation = "c29ec348-61ec-40c8-8164-b8c60e9d9f3d" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" -BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" ComputationalResources = "ed09eef8-17a6-5b46-8889-db040fac31e3" diff --git a/test/benchmark_tests.jl b/test/benchmark_tests.jl deleted file mode 100644 index 3939af10..00000000 --- a/test/benchmark_tests.jl +++ /dev/null @@ -1,45 +0,0 @@ -@testset "Benchmark" begin - BenchmarkTools.DEFAULT_PARAMETERS.samples = 2^13 - BenchmarkTools.DEFAULT_PARAMETERS.seconds = convert(Float64, 2 * 60) - BenchmarkTools.DEFAULT_PARAMETERS.evals = 1 - BenchmarkTools.DEFAULT_PARAMETERS.overhead = 0.0 - BenchmarkTools.DEFAULT_PARAMETERS.gctrial = true - BenchmarkTools.DEFAULT_PARAMETERS.gcsample = false - BenchmarkTools.DEFAULT_PARAMETERS.time_tolerance = convert(Float64, eps(Float16)) - BenchmarkTools.DEFAULT_PARAMETERS.memory_tolerance = convert(Float64, eps(Float16)) - - nvars = 8 - n = 128 - rng = Random.default_rng() - r = rand(Float32, nvars, n) - - nn = FluxCompatLayer(Flux.f32(Flux.Dense(nvars => nvars, tanh))) - icnf = construct(RNODE, nn, nvars; compute_mode = ZygoteMatrixMode) - - ps, st = Lux.setup(rng, icnf) - diff_loss_train(x) = loss(icnf, TrainMode(), r, x, st) - diff_loss_test(x) = loss(icnf, TestMode(), r, x, st) - grad_diff_loss_train() = Zygote.gradient(diff_loss_train, ps) - grad_diff_loss_test() = Zygote.gradient(diff_loss_test, ps) - t_loss_train() = loss(icnf, TrainMode(), r, ps, st) - t_loss_test() = loss(icnf, TestMode(), r, ps, st) - - ben_loss_train = BenchmarkTools.@benchmark $t_loss_train() - ben_loss_test = BenchmarkTools.@benchmark $t_loss_test() - ben_grad_train = BenchmarkTools.@benchmark $grad_diff_loss_train() - ben_grad_test = BenchmarkTools.@benchmark $grad_diff_loss_test() - - @info "t_loss_train" - display(ben_loss_train) - - @info "t_loss_test" - display(ben_loss_test) - - @info "grad_diff_loss_train" - display(ben_grad_train) - - @info "grad_diff_loss_test" - display(ben_grad_test) - - @test true -end diff --git a/test/runtests.jl b/test/runtests.jl index 972286c3..3db21dd5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,7 +3,6 @@ using Test using AbstractDifferentiation: AbstractDifferentiation using ADTypes: ADTypes using Aqua: Aqua -using BenchmarkTools: BenchmarkTools using ComponentArrays: ComponentArrays using ComputationalResources: ComputationalResources using CUDA: CUDA @@ -42,10 +41,6 @@ USE_GPU = get(ENV, "USE_GPU", "Yes") == "Yes" end end - if GROUP == "All" || GROUP == "Benchmark" - include("benchmark_tests.jl") - end - if GROUP == "All" || GROUP == "Quality" include("quality_tests.jl") end