diff --git a/docs/Manifest.toml b/docs/Manifest.toml index 0cff207..e07bcfc 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -216,15 +216,15 @@ version = "0.1.2" [[deps.ConstraintCommons]] deps = ["Dictionaries", "TestItemRunner", "TestItems"] -git-tree-sha1 = "866bd8d55f57037d7542c9f1d8d17285f44b78dd" +git-tree-sha1 = "4698469c1292bea5573dc294a180c364d6ebfa81" repo-rev = "main" repo-url = "https://github.com/JuliaConstraints/ConstraintCommons.jl.git" uuid = "e37357d9-0691-492f-a822-e5ea6a920954" -version = "0.2.0" +version = "0.2.1" [[deps.ConstraintDomains]] deps = ["ConstraintCommons", "Dictionaries", "Intervals", "PatternFolds", "StatsBase", "TestItemRunner", "TestItems"] -git-tree-sha1 = "0d087d2580392660948480a0484dbb17b0141e83" +git-tree-sha1 = "50801b71002e4b8c7d943980a8b3008dcbc2aa33" repo-rev = "main" repo-url = "https://github.com/JuliaConstraints/ConstraintDomains.jl.git" uuid = "5800fd60-8556-4464-8d61-84ebf7a0bedb" @@ -374,9 +374,9 @@ version = "1.4.1" [[deps.DocumenterVitepress]] deps = ["ANSIColoredPrinters", "Base64", "DocStringExtensions", "Documenter", "IOCapture", "Markdown", "NodeJS_20_jll", "REPL"] -git-tree-sha1 = "733cf5d8819a583b2753a59e820e9de130c2df4e" +git-tree-sha1 = "a58b1681c9d3ef9f5842ac86c13f9f5b8ff2eee6" uuid = "4710194d-e776-4893-9690-8d956a29c365" -version = "0.1.0" +version = "0.1.1" [deps.DocumenterVitepress.extensions] DocumenterVitepressDocumenterCitationsExt = "DocumenterCitations" @@ -461,9 +461,9 @@ version = "2.23.1" [[deps.Flux]] deps = ["Adapt", "ChainRulesCore", "Compat", "Functors", "LinearAlgebra", "MLUtils", "MacroTools", "NNlib", "OneHotArrays", "Optimisers", "Preferences", "ProgressLogging", "Random", "Reexport", "SparseArrays", "SpecialFunctions", "Statistics", "Zygote"] -git-tree-sha1 = "a5475163b611812d073171583982c42ea48d22b0" +git-tree-sha1 = "edacf029ed6276301e455e34d7ceeba8cc34078a" uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c" -version = "0.14.15" +version = "0.14.16" [deps.Flux.extensions] FluxAMDGPUExt = "AMDGPU" @@ -499,9 +499,9 @@ uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" [[deps.GPUArrays]] deps = ["Adapt", "GPUArraysCore", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "Serialization", "Statistics"] -git-tree-sha1 = "38cb19b8a3e600e509dc36a6396ac74266d108c1" +git-tree-sha1 = "c154546e322a9c73364e8a60430b0f79b812d320" uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "10.1.1" +version = "10.2.0" [[deps.GPUArraysCore]] deps = ["Adapt"] @@ -528,9 +528,9 @@ version = "1.3.1" [[deps.IOCapture]] deps = ["Logging", "Random"] -git-tree-sha1 = "8b72179abc660bfab5e28472e019392b97d0985c" +git-tree-sha1 = "b6d6bfdd7ce25b0f9b2f6b3dd56b2673a66c8770" uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" -version = "0.2.4" +version = "0.2.5" [[deps.IRTools]] deps = ["InteractiveUtils", "MacroTools"] @@ -550,9 +550,15 @@ version = "0.3.1" [[deps.InlineStrings]] deps = ["Parsers"] -git-tree-sha1 = "9cc2baf75c6d09f9da536ddf58eb2f29dedaf461" +git-tree-sha1 = "86356004f30f8e737eff143d57d41bd580e437aa" uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" -version = "1.4.0" +version = "1.4.1" + + [deps.InlineStrings.extensions] + ArrowTypesExt = "ArrowTypes" + + [deps.InlineStrings.weakdeps] + ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd" [[deps.InteractiveUtils]] deps = ["Markdown"] @@ -593,9 +599,9 @@ version = "0.21.4" [[deps.JuMP]] deps = ["LinearAlgebra", "MacroTools", "MathOptInterface", "MutableArithmetics", "OrderedCollections", "PrecompileTools", "Printf", "SparseArrays"] -git-tree-sha1 = "28f9313ba6603e0d2850fc3eae617e769c99bf83" +git-tree-sha1 = "7e10a0d8b534f2d8e9f712b33488584254624fb1" uuid = "4076af6c-e467-56ae-b986-b466b2749572" -version = "1.22.1" +version = "1.22.2" [deps.JuMP.extensions] JuMPDimensionalDataExt = "DimensionalData" @@ -813,9 +819,9 @@ version = "2023.1.10" [[deps.MutableArithmetics]] deps = ["LinearAlgebra", "SparseArrays", "Test"] -git-tree-sha1 = "a3589efe0005fc4718775d8641b2de9060d23f73" +git-tree-sha1 = "898c56fbf8bf71afb0c02146ef26f3a454e88873" uuid = "d8a4904e-b15c-11e9-3269-09a3773c0cb0" -version = "1.4.4" +version = "1.4.5" [[deps.NLSolversBase]] deps = ["DiffResults", "Distributed", "FiniteDiff", "ForwardDiff"] @@ -930,8 +936,8 @@ version = "0.2.4" [[deps.PerfChecker]] deps = ["CSV", "CoverageTools", "CpuId", "JSON", "Malt", "Pkg", "Profile", "TOML", "TypedTables", "UUIDs"] -git-tree-sha1 = "113b402af7d1d4d1d1a4fe176f7d4881822872b1" -repo-rev = "main" +git-tree-sha1 = "58979ea0b3de20e82626de04aca6328e1a81c3fd" +repo-rev = "docstuff" repo-url = "https://github.com/JuliaConstraints/PerfChecker.jl.git" uuid = "6309bf6b-a531-4b08-891e-8ee981e5c424" version = "0.2.1" diff --git a/docs/make.jl b/docs/make.jl index 4bffbc8..c64b869 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -38,7 +38,8 @@ makedocs(; repo = "https://github.com/JuliaConstraints/JuliaConstraints.github.io", devurl = "dev", deploy_url = "https://juliaconstraints.github.io/", - description = "Resources about Constraint Programming in Julia" + description = "Resources about Constraint Programming in Julia", + build_vitepress = false ), pages=[ "Home" => "index.md", @@ -106,8 +107,11 @@ makedocs(; ], "Performance Checking" => [ "PerfChecker.jl" => "perf/perf_checker.md", - "BenchmarkToolsExt" => "perf/benchmark_ext.md", + "Tutorial" => "perf/tutorial.md", "Interface" => "perf/perf_interface.md", + "BenchmarkToolsExt" => "perf/benchmark_ext.md", + "ChairmarksExt" => "perf/chairmarks_ext.md", + "API" => "perf/api.md" ], "API" => [ "Public" => "public_api.md", diff --git a/docs/src/perf/api.md b/docs/src/perf/api.md new file mode 100644 index 0000000..8695b70 --- /dev/null +++ b/docs/src/perf/api.md @@ -0,0 +1,7 @@ +# API + +Here's the API for PerfChecker.jl + +```@autodocs +Modules=[PerfChecker] +``` diff --git a/docs/src/perf/benchmark_ext.md b/docs/src/perf/benchmark_ext.md index bc7dbb8..ab6a780 100644 --- a/docs/src/perf/benchmark_ext.md +++ b/docs/src/perf/benchmark_ext.md @@ -1,3 +1,36 @@ # BenchmarkTools Extension -A benchmarking extension, based on `BenchmarkTools.jl`, has been interfaced with `PerfChecker.jl`. This section (will) provides some usage examples, documentation, and links to related notebooks. +A benchmarking extension, based on `BenchmarkTools.jl`, has been interfaced with `PerfChecker.jl`. +This section will provide some usage examples, documentation, and links to related notebooks. + +## Usage + +Like all other extensions, `BenchmarkTools` extension can be used in the following way: + +```julia +julia> using BenchmarkTools, PerfChecker + +julia> @check :benchmark Dict(:option1 => "value1", :option2 => "value2", :PATH => @__DIR__) begin + # the prelimnary code goes here + using Example +end begin + # the code you want to be benchmarked + Example.domath(10) # returns x + 5, 15 in this case. +end +``` + +## Options + +Options specific to this backend with their default values are defined as: +```julia +:threads => 1 +:track => "none" +:samples => BenchmarkTools.DEFAULT_PARAMETERS.samples +:seconds => BenchmarkTools.DEFAULT_PARAMETERS.seconds +:evals => BenchmarkTools.DEFAULT_PARAMETERS.evals +:overhead => BenchmarkTools.DEFAULT_PARAMETERS.overhead +:gctrial => BenchmarkTools.DEFAULT_PARAMETERS.gctrial +:gcsample => BenchmarkTools.DEFAULT_PARAMETERS.gcsample +:time_tolerance => BenchmarkTools.DEFAULT_PARAMETERS.time_tolerance +:memory_tolerance => BenchmarkTools.DEFAULT_PARAMETERS.memory_tolerance +``` diff --git a/docs/src/perf/chairmarks_ext.md b/docs/src/perf/chairmarks_ext.md new file mode 100644 index 0000000..166a17a --- /dev/null +++ b/docs/src/perf/chairmarks_ext.md @@ -0,0 +1,32 @@ +# Chairmarks Extension + +A benchmarking extension, based on `Chairmarks.jl`, has been interfaced with `PerfChecker.jl`. +This section will provide some usage examples, documentation, and links to related notebooks. + +## Usage + +Like all other extensions, `BenchmarkTools` extension can be used in the following way: + +```julia +julia> using Chairmarks, PerfChecker + +julia> @check :chairmark Dict(:option1 => "value1", :option2 => "value2", :PATH => @__DIR__) begin + # the prelimnary code goes here + using Example +end begin + # the code you want to be benchmarked + Example.domath(10) # returns x + 5, 15 in this case. +end +``` + +## Options + +Options specific to this backend with their default values are defined as: +```julia +:threads => 1 +:track => "none" +:evals => nothing +:seconds => 1, +:samples => nothing +:gc => true +``` diff --git a/docs/src/perf/perf_checker.md b/docs/src/perf/perf_checker.md index 8a8f40a..4906f5a 100644 --- a/docs/src/perf/perf_checker.md +++ b/docs/src/perf/perf_checker.md @@ -1,7 +1,48 @@ # PerfChecker.jl -Documentation for `PerfChecker.jl`. +PerfChecker.jl is a package designed for package authors to easily performance test their packages. +To achieve that, it provides the follwing features: -```@autodocs -Modules=[PerfChecker] +- The main macro `@check`, which provides an easy-to-use interface over various interfaces, configurable for various backends via a dictionary. +- (WIP) A CI for reproducible performance testing. +- Visualization of different metrics from `@check` using Makie.jl + +## Usage + +The primary usage of PerfChecker.jl looks like this: + +```julia + using PerfChecker + # optional using custom backend like BenchmarkTools, Chairmark etc + config = Dict(:option1 => "value1", :option2 => :value2) + + results = @check :name_of_backend config begin + # preparatory code goes here + end begin + # the code block to be performance tested goes here + end + + # Visualization of the results + using Makie + checkres_to_scatterlines(results) ``` + +The config dictionary can take many options, depending on the backend. + +Some of the commonly used options are: +- `:PATH` => The path where to the default environment of julia when creating a new process. +- `:pkgs` => A list of versions to test performance for. Its defined as the `Tuple`, `(name::String, option::Symbol, versions::Vector{VersionNumber}, last_or_first::Bool)` Can be given as follows: + - `name` is the name of the package. + - `option` is one of the 5 symbols: + - `:patches`: last patch or first patch of a version + - `:breaking`: last breaking or next breaking version + - `:major`: previous or next major version + - `:minor`: previous or next minor version + - `:custom`: custom version numbers (provide any boolean value for `last_or_first` in this case as it doesn't matter) + - `versions`: The input for the provided `option` + - `last_or_first`: Input for the provided `option` +- `:tags` => A list of tags (a vector of symbols) to easily tag performance tests. +- `:devops` => Giving a custom input to `Pkg.develop`. Intended to be used to test performance of a local development branch of a pacakge with previous versions. Often can be used as simply as `:devops => "MyPackageName"` +- `:threads` => An integer to select the number of threads to start Julia with. + +Checkout the documentation of the other backends for more default options and the default values. diff --git a/docs/src/perf/perf_interface.md b/docs/src/perf/perf_interface.md index 986949c..b59d826 100644 --- a/docs/src/perf/perf_interface.md +++ b/docs/src/perf/perf_interface.md @@ -1,3 +1,101 @@ -# Interfacing PerfChecker +# Extending PerfChecker -PerfChecker was build as an easy to extend interface. This section will cover the few method required. +PerfChecker was build as an easy to extend interface. A good reference example for this is the Chairmarks extension. + +Extending PerfChecker works via PkgExtensions feature in Julia. There are 6 essential functions that need to be extended inside the Pkg extension. +Each extension has a keyword symbol for it, which users can input to use the extension. + +## The Default Options + +Method to be overloaded: `PerfChecker.default_options(::Val{:myperfextension})::Dict` + +PerfChecker works via a config dictionary. Users can populate this dictionary with options and provide it to the main `check` macro to customize the performance testing to their liking. + +For Chairmarks.jl, it looks like this: +```julia +function PerfChecker.default_options(::Val{:chairmark}) + return Dict( + :threads => 1, + :track => "none", + :evals => nothing, + :seconds => 1, + :samples => nothing, + :gc => true + ) +end +``` + + +## Package initialization +Method to be overloaded: `PerfChecker.initpkgs(::Val{:myperfextension})::Expr` + +This method is plainly to load the main package(s) associated with the custom backend. In case of Chairmarks.jl, it looks like this: +```julia +PerfChecker.initpkgs(::Val{:chairmark}) = quote + using Chairmarks +end +``` + +## Preparatory Code: +Method to be overloaded: `PerfChecker.prep(config_dict::Dict, block::Expr, ::Val{:myperfextension})::Expr` + +This method exists to run arbitrary "preparatory" code (represented by `block` parameter here) before running the code to be performance tested for. + +The output from here are stored inside the `:prep_result` key of the configuration dictionary. + +Example for Chairmarks.jl: +```julia +PerfChecker.prep(::Dict, block::Expr, ::Val{:chairmark}) = quote + $block + nothing +end +``` + +This just runs the code in `block` provided by the user. + +## Main Code to be Performance Tested +Method to be overloaded: `PerfChecker.check(config_dict::Dict, block::Expr, ::Val{:myperfextension})::Expr`. + +Runs the appropriate code to run the performance testing on user's code. For Chairmarks.jl, it looks like this: +```julia +function PerfChecker.check(d::Dict, block::Expr, ::Val{:chairmark}) + quote + d = $d + return @be $block evals=d[:evals] seconds=d[:seconds] samples=d[:samples] gc=d[:gc] + end +end +``` + +The output from here are stored inside the `:check_result` key of the configuration dictionary. + +## Post Performance Testing Code +Method to be overloaded: `PerfChecker.post(config_dict::Dict, ::Val{:myperfextension})` + +The code to be run after the performance testing is done. The output from here is converted into a table via the `to_table` method overloading. + +In the case of Chairmarks.jl: +```julia +PerfChecker.post(d::Dict, ::Val{:chairmark}) = d[:check_result] +``` + +## Converting the result into a Table +Method to be overloaded: `PerfChecker.to_table` + +Convert the output from `post` function into an appropriate table. + +In the case of Chairmarks.jl +```julia +function PerfChecker.to_table(chair::Chairmarks.Benchmark) + l = length(chair.samples) + times = [chair.samples[i].time for i in 1:l] + gctimes = [chair.samples[i].gc_fraction for i in 1:l] + bytes = [chair.samples[i].bytes for i in 1:l] + allocs = [chair.samples[i].allocs for i in 1:l] + return Table(times = times, gctimes = gctimes, bytes = bytes, allocs = allocs) +end +``` + + +--- + +There are also other functions that can be overloaded, mostly related to plotting but these are the basic functions to extend PerfChecker for a custom backend. diff --git a/docs/src/perf/tutorial.md b/docs/src/perf/tutorial.md new file mode 100644 index 0000000..f72dbcf --- /dev/null +++ b/docs/src/perf/tutorial.md @@ -0,0 +1,68 @@ +# Tutorial + +Taken from PerfChecker.jl examples, this is a guide for performance testing of PatterFolds.jl package using Chairmarks.jl + +Using PerfChecker.jl requires an environment with the dependencies present in it. + +The actual script looks like this: + +```julia +using PerfChecker, Chairmarks, CairoMakie + +d = Dict(:path => @__DIR__, :evals => 10, :samples => 1000, + :seconds => 100, :tags => [:patterns, :intervals], + :pkgs => ( + "PatternFolds", :custom, [v"0.2.0", v"0.2.1", v"0.2.2", v"0.2.3", v"0.2.4"], true), + :devops => "PatternFolds") + +x = @check :chairmark d begin + using PatternFolds +end begin + # Intervals + itv = Interval{Open, Closed}(0.0, 1.0) + i = IntervalsFold(itv, 2.0, 1000) + + unfold(i) + collect(i) + reverse(collect(i)) + + # Vectors + vf = make_vector_fold([0, 1], 2, 1000) + + unfold(vf) + collect(vf) + reverse(collect(vf)) + + rand(vf, 1000) + + return nothing +end + +mkpath(joinpath(@__DIR__, "visuals")) + +c = checkres_to_scatterlines(x, Val(:chairmark)) +save(joinpath(@__DIR__, "visuals", "chair_evolution.png"), c) + +for kwarg in [:times, :gctimes, :bytes, :allocs] + c2 = checkres_to_boxplots(x, Val(:chairmark); kwarg) + save(joinpath(@__DIR__, "visuals", "chair_boxplots_$kwarg.png"), c2) +end +``` + +`d` here is the configuration dictionary. +`x` stores the results from performance testing + +The code below the macro call is for plotting and storing the plots. It creates the visuals folder and stores the following plots in the folder: + +Boxplots from Chairmarks for allocations: + +![chair_boxplots](https://github.com/JuliaConstraints/PerfChecker.jl/blob/main/perf/PatternFolds/visuals/chair_boxplots_allocs.png?raw=true) + +Boxplots from Chairmarks for times: + +![chair_times](https://github.com/JuliaConstraints/PerfChecker.jl/blob/main/perf/PatternFolds/visuals/chair_boxplots_times.png?raw=true) + +Evolution of different metrics across versions according to Chairmarks: + +![chair_evolution](https://github.com/JuliaConstraints/PerfChecker.jl/blob/main/perf/PatternFolds/visuals/chair_evolution.png?raw=true) +