diff --git a/.github/workflows/push_preview_cleanup.yml b/.github/workflows/push_preview_cleanup.yml new file mode 100644 index 0000000..5fa88a3 --- /dev/null +++ b/.github/workflows/push_preview_cleanup.yml @@ -0,0 +1,27 @@ +# from https://juliadocs.github.io/Documenter.jl/dev/man/hosting/#gh-pages-Branch +name: Doc Preview Cleanup +on: + pull_request: + types: [closed] +jobs: + doc-preview-cleanup: + runs-on: ubuntu-latest + steps: + - name: Checkout gh-pages branch + uses: actions/checkout@v2 + with: + ref: gh-pages + + - name: Delete preview and history + run: | + git config user.name "Documenter.jl" + git config user.email "documenter@juliadocs.github.io" + git rm -rf "previews/PR$PRNUM" + git commit -m "delete preview" + git branch gh-pages-new $(echo "delete history" | git commit-tree HEAD^{tree}) + env: + PRNUM: ${{ github.event.number }} + + - name: Push changes + run: | + git push --force origin gh-pages-new:gh-pages diff --git a/docs/make.jl b/docs/make.jl index ca5f913..768fae9 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -3,7 +3,9 @@ using Documenter makedocs(; modules=[Lighthouse], sitename="Lighthouse", authors="Beacon Biosignals and other contributors", - pages=["API Documentation" => "index.md"]) + pages=["API Documentation" => "index.md", + "Terminology" => "terminology.md", + "Evaluation metrics" => "evaluation_metrics.md"]) deploydocs(repo="github.com/beacon-biosignals/Lighthouse.jl.git", - devbranch="main") + devbranch="main", push_preview=true) diff --git a/docs/src/evaluation_metrics.md b/docs/src/evaluation_metrics.md new file mode 100644 index 0000000..9a83447 --- /dev/null +++ b/docs/src/evaluation_metrics.md @@ -0,0 +1,28 @@ +# Evaluation metrics + +Lighthouse automatically generates a suite of evaluation metrics. +Here, we briefly describe these. This page uses terms defined in [Terminology](@ref), +so see that page for any unfamiliar words. + +## Confusion matrices + +Lighthouse plots confusion matrices, which are simple tables +showing the empirical distribution of predicted class (the rows) +versus the elected class (the columns). These come in two variants: + +* row-normalized: this means each row has been normalized to sum to 1. Thus, the row-normalized confusion matrix shows the empirical distribution of elected classes for a given predicted class. E.g. the first row of the row-normalized confusion matrix shows the empirical probabilities of the elected classes for a sample which was predicted to be in the first class. +* column-normalized: this means each column has been normalized to sum to 1. Thus, the column-normalized confusion matrix shows the empirical distribution of predicted classes for a given elected class. E.g. the first column of the column-normalized confusion matrix shows the empirical probabilities of the predicted classes for a sample which was elected to be in the first class. + +[insert example plot] + +## Inter-rater reliability + +## ROC curves + +## PR curves (precision-recall curves) + +## PR-gain curves (precision-recall-gain curves) + +## Prediction-reliability calibration + +## diff --git a/docs/src/terminology.md b/docs/src/terminology.md new file mode 100644 index 0000000..ec5466f --- /dev/null +++ b/docs/src/terminology.md @@ -0,0 +1,9 @@ +# Terminology + +* _sample_: a piece of data to be classified by the model, or a labelled piece of training/test/validation data. +* _classes_: the set of possible class labels which the model attempts to predict. +* _voters_: the individual sources of labelled data, such as human labellers. Each voter may supply a "vote" for a class label for a sample. +* _votes_: the matrix of votes corresponding to a set of data, whose rows correspond to the index of a sample in a set of data, whose columns correspond to voters, and whose values are the indices of class labels (i.e. numbers in `1:length(classes)`). E.g. if 2 voters have voted on ten samples, then `votes` is a 10 by 2 matrix of integers. If a voter has not voted on a particular sample, any value outside `1:length(classes)` may be supplied to indicate this. +* _elected class_: the class elected by the voters. By default in [`learn!`](@ref), + the elected class is chosen by a simple majority of the votes with ties broken randomly. +* _predicted class_: the class predicted by the model for a given input. diff --git a/src/Lighthouse.jl b/src/Lighthouse.jl index 6795ff8..45ef8a6 100644 --- a/src/Lighthouse.jl +++ b/src/Lighthouse.jl @@ -9,8 +9,10 @@ using StatsPlots.Plots.PlotMeasures using Printf # Set up plotting backend for Plots.jl (GR) -gr() -GR.inline("png") +function __init__() + gr() + GR.inline("png") +end include("utilities.jl") export majority diff --git a/src/learn.jl b/src/learn.jl index 93ff9c3..2a0977c 100644 --- a/src/learn.jl +++ b/src/learn.jl @@ -19,6 +19,11 @@ struct LearnLogger logged::Dict{String,Vector{Any}} end +""" + LearnLogger(path, run_name; kwargs...) + +Creates a `LearnLogger` which stores tensorboard logs at `path` under the subdirectory `run_name`. +""" function LearnLogger(path, run_name; kwargs...) tensorboard_logger = TBLogger(joinpath(path, run_name); kwargs...) return LearnLogger(path, tensorboard_logger, Dict{String,Any}()) @@ -124,16 +129,14 @@ The following quantities are logged to `logger`: Where... -- `model` is a model that outputs soft labels when called on a batch of `batches`, -`model(batch)`. +- `model` is a model that outputs soft labels when called on a batch of `batches`, `model(batch)`. - `predicted_soft_labels` is a matrix whose columns correspond to classes and -whose rows correspond to samples in batches, and which is filled in with soft-label -predictions. + whose rows correspond to samples in batches, and which is filled in with soft-label predictions. - `batches` is an iterable of batches, where each element of -the iterable takes the form `(batch, votes_locations)`. Internally, `batch` is -passed to [`loss_and_prediction`](@ref) as `loss_and_prediction(model, batch...)`. + the iterable takes the form `(batch, votes_locations)`. Internally, `batch` is + passed to [`loss_and_prediction`](@ref) as `loss_and_prediction(model, batch...)`. """ function predict!(model::AbstractClassifier, predicted_soft_labels::AbstractMatrix, batches, @@ -180,27 +183,27 @@ The following quantities are logged to `logger`: Where... - `predicted_soft_labels` is a matrix of soft labels whose columns correspond to -classes and whose rows correspond to samples in the evaluation set. + classes and whose rows correspond to samples in the evaluation set. - `predicted_hard_labels` is a vector of hard labels where the `i`th element -is the hard label predicted by the model for sample `i` in the evaulation set. + is the hard label predicted by the model for sample `i` in the evaulation set. - `elected_hard_labels` is a vector of hard labels where the `i`th element -is the hard label elected as "ground truth" for sample `i` in the evaulation set. + is the hard label elected as "ground truth" for sample `i` in the evaulation set. - `thresholds` are the range of thresholds used by metrics (e.g. PR curves) that -are calculated on the `predicted_soft_labels` for a range of thresholds. + are calculated on the `predicted_soft_labels` for a range of thresholds. - `votes` is a matrix of hard labels whose columns correspond to voters and whose -rows correspond to the samples in the test set that have been voted on. If -`votes[sample, voter]` is not a valid hard label for `model`, then `voter` will -simply be considered to have not assigned a hard label to `sample`. + rows correspond to the samples in the test set that have been voted on. If + `votes[sample, voter]` is not a valid hard label for `model`, then `voter` will + simply be considered to have not assigned a hard label to `sample`. - `optimal_threshold_class` is the class index (`1` or `2`) for which to calculate -an optimal threshold for converting the `predicted_soft_labels` to -`predicted_hard_labels`. If present, the input `predicted_hard_labels` will be -ignored and new `predicted_hard_labels` will be recalculated from the new threshold. -This is only a valid parameter when `length(classes) == 2` + an optimal threshold for converting the `predicted_soft_labels` to + `predicted_hard_labels`. If present, the input `predicted_hard_labels` will be + ignored and new `predicted_hard_labels` will be recalculated from the new threshold. + This is only a valid parameter when `length(classes) == 2` """ function evaluate!(predicted_hard_labels::AbstractVector, predicted_soft_labels::AbstractMatrix, @@ -217,6 +220,7 @@ function evaluate!(predicted_hard_labels::AbstractVector, votes=votes, optimal_threshold_class=optimal_threshold_class) log_plot!(logger, logger_prefix * "/metrics" * logger_suffix, plot, plot_data) + log_value!(logger, logger_prefix * "/metrics/accuracy" * logger_suffix, accuracy(plot_data["confusion_matrix"])) if haskey(plot_data, "spearman_correlation") log_value!(logger, logger_prefix * "/spearman_correlation" * logger_suffix, plot_data["spearman_correlation"].ρ) @@ -250,22 +254,26 @@ function plot_roc_curves(per_class_roc_curves, per_class_roc_aucs, class_labels; end function plot_reliability_calibration_curves(per_class_reliability_calibration_curves, - per_class_reliability_calibration_scores, - class_labels; legend=:bottomright) + per_class_reliability_calibration_scores, + class_labels; legend=:bottomright) + + eltype(per_class_reliability_calibration_scores) >: Missing && return plot() + calibration_score_labels = [@sprintf("%s (MSE: %.3f)", class, - per_class_reliability_calibration_scores[i]) - for (i, class) in enumerate(class_labels)] + per_class_reliability_calibration_scores[i]) + for (i, class) in enumerate(class_labels)] plot(per_class_reliability_calibration_curves; labels=calibration_score_labels, - title="Prediction reliability calibration", xlabel="Predicted probability bin", - xlims=(0, 1), ylabel="Fraction of positives", ylims=(0, 1), markershape=:circle, - markersize=2, linewidth=1, markerstrokewidth=0, legendfontsize=1, legend=legend, - framestyle=:box) + title="Prediction reliability calibration", xlabel="Predicted probability bin", + xlims=(0, 1), ylabel="Fraction of positives", ylims=(0, 1), markershape=:circle, + markersize=2, linewidth=1, markerstrokewidth=0, legendfontsize=1, legend=legend, + framestyle=:box) #TODO: mean predicted value histogram underneath?? Maybe important... # https://scikit-learn.org/stable/modules/calibration.html plot!([0, 1], [0, 1]; linecolor=:black, linestyle=:dash, label="Ideal") return xticks!(0:0.2:1) end + function plot_binary_discrimination_calibration_curves(calibration_curve, calibration_score, per_expert_calibration_curves, per_expert_calibration_scores, @@ -453,11 +461,9 @@ for class `i`. Where... - `predicted_hard_labels` is a vector of hard labels where the `i`th element -is the hard label predicted by the model for sample `i` in the evaulation set. - + is the hard label predicted by the model for sample `i` in the evaulation set. - `elected_hard_labels` is a vector of hard labels where the `i`th element is the hard label elected as "ground truth" for sample `i` in the evaulation set. - - `class_count` is the number of possible classes. """ @@ -485,9 +491,9 @@ IRA kappa calculated for class `i`. Where... - `votes` is a matrix of hard labels whose columns correspond to voters and whose -rows correspond to the samples in the test set that have been voted on. If -`votes[sample, voter]` is not a valid hard label for `model`, then `voter` will -simply be considered to have not assigned a hard label to `sample`. + rows correspond to the samples in the test set that have been voted on. If + `votes[sample, voter]` is not a valid hard label for `model`, then `voter` will + simply be considered to have not assigned a hard label to `sample`. - `classes` all possible classes voted on. @@ -552,16 +558,14 @@ Only valid for binary classification problems (i.e., `length(classes) == 2`) Where... - `predicted_soft_labels` is a matrix of soft labels whose columns correspond to -the two classes and whose rows correspond to the samples in the test set that have been -classified. For a given sample, the two class column values must sum to 1 (i.e., -softmax has been applied to the classification output). - + the two classes and whose rows correspond to the samples in the test set that have been + classified. For a given sample, the two class column values must sum to 1 (i.e., + softmax has been applied to the classification output). - `votes` is a matrix of hard labels whose columns correspond to voters and whose -rows correspond to the samples in the test set that have been voted on. If + rows correspond to the samples in the test set that have been voted on. If `votes[sample, voter]` is not a valid hard label for `model`, then `voter` will -simply be considered to have not assigned a hard label to `sample`. May contain -a single voter (i.e., a single column). - + simply be considered to have not assigned a hard label to `sample`. May contain + a single voter (i.e., a single column). - `classes` are the two classes voted on. """ function _calculate_spearman_correlation(predicted_soft_labels, votes, classes) @@ -678,19 +682,19 @@ is the hard label elected as "ground truth" for sample `i` in the evaulation set are calculated on the `predicted_soft_labels` for a range of thresholds. - `votes` is a matrix of hard labels whose columns correspond to voters and whose -rows correspond to the samples in the test set that have been voted on. If -`votes[sample, voter]` is not a valid hard label for `model`, then `voter` will -simply be considered to have not assigned a hard label to `sample`. + rows correspond to the samples in the test set that have been voted on. If + `votes[sample, voter]` is not a valid hard label for `model`, then `voter` will + simply be considered to have not assigned a hard label to `sample`. - `strata` is a vector of sets of (arbitrarily typed) groups/strata for each sample -in the evaluation set, or `nothing`. If not `nothing`, per-class and multiclass -kappas will also be calculated per group/stratum. + in the evaluation set, or `nothing`. If not `nothing`, per-class and multiclass + kappas will also be calculated per group/stratum. - `optimal_threshold_class` is the class index (`1` or `2`) for which to calculate -an optimal threshold for converting the `predicted_soft_labels` to -`predicted_hard_labels`. If present, the input `predicted_hard_labels` will be -ignored and new `predicted_hard_labels` will be recalculated from the new threshold. -This is only a valid parameter when `length(classes) == 2` + an optimal threshold for converting the `predicted_soft_labels` to + `predicted_hard_labels`. If present, the input `predicted_hard_labels` will be + ignored and new `predicted_hard_labels` will be recalculated from the new threshold. + This is only a valid parameter when `length(classes) == 2` """ function evaluation_metrics_plot(predicted_hard_labels::AbstractVector, predicted_soft_labels::AbstractMatrix, @@ -840,7 +844,7 @@ end learn!(model::AbstractClassifier, logger, get_train_batches, get_test_batches, votes, elected=majority.(eachrow(votes), (1:length(classes(model)),)); - epoch_limit=100, post_epoch_callback=(_ -> nothing), + epoch_limit=100, post_epoch_callback=(current_epoch -> nothing), optimal_threshold_class::Union{Nothing,Integer}=nothing) Return `model` after optimizing its parameters across multiple epochs of @@ -875,34 +879,34 @@ of logged values, `\$resource` takes the values of the field names of Where... - `get_train_batches` is a zero-argument function that returns an iterable of -training set batches. Internally, `learn!` uses this function when it calls -`train!(model, get_train_batches(), logger)`. + training set batches. Internally, `learn!` uses this function when it calls + `train!(model, get_train_batches(), logger)`. - `get_test_batches` is a zero-argument function that returns an iterable -of test set batches used during the current epoch's test phase. Each element of -the iterable takes the form `(batch, votes_locations)`. Internally, `batch` is -passed to [`loss_and_prediction`](@ref) as `loss_and_prediction(model, batch...)`, -and `votes_locations[i]` is expected to yield the row index of `votes` that -corresponds to the `i`th sample in `batch`. + of test set batches used during the current epoch's test phase. Each element of + the iterable takes the form `(batch, votes_locations)`. Internally, `batch` is + passed to [`loss_and_prediction`](@ref) as `loss_and_prediction(model, batch...)`, + and `votes_locations[i]` is expected to yield the row index of `votes` that + corresponds to the `i`th sample in `batch`. - `votes` is a matrix of hard labels whose columns correspond to voters and whose -rows correspond to the samples in the test set that have been voted on. If -`votes[sample, voter]` is not a valid hard label for `model`, then `voter` will -simply be considered to have not assigned a hard label to `sample`. + rows correspond to the samples in the test set that have been voted on. If + `votes[sample, voter]` is not a valid hard label for `model`, then `voter` will + simply be considered to have not assigned a hard label to `sample`. - `elected` is a vector of hard labels where the `i`th element is the hard label -elected as "ground truth" out of `votes[i, :]`. + elected as "ground truth" out of `votes[i, :]`. - `optimal_threshold_class` is the class index (`1` or `2`) for which to calculate -an optimal threshold for converting `predicted_soft_labels` to `predicted_hard_labels`. -This is only a valid parameter when `length(classes) == 2`. If `optimal_threshold_class` -is present, test set evaluation will be based on predicted hard labels calculated -with this threshold; if `optimal_threshold_class` is `nothing`, predicted hard labels -will be calculated via `onecold(classifier, soft_label)`. + an optimal threshold for converting `predicted_soft_labels` to `predicted_hard_labels`. + This is only a valid parameter when `length(classes) == 2`. If `optimal_threshold_class` + is present, test set evaluation will be based on predicted hard labels calculated + with this threshold; if `optimal_threshold_class` is `nothing`, predicted hard labels + will be calculated via `onecold(classifier, soft_label)`. """ function learn!(model::AbstractClassifier, logger, get_train_batches, get_test_batches, votes, elected=majority.(eachrow(votes), (1:length(classes(model)),)); - epoch_limit=100, post_epoch_callback=(_ -> nothing), + epoch_limit=100, post_epoch_callback=(current_epoch -> nothing), optimal_threshold_class::Union{Nothing,Integer}=nothing) # NOTE `votes` is currently unused except to construct `elected` by default, # but will be necessary for calculating multirater metrics e.g. Fleiss' kappa diff --git a/src/utilities.jl b/src/utilities.jl index 5e1ea71..ee80df8 100644 --- a/src/utilities.jl +++ b/src/utilities.jl @@ -16,7 +16,8 @@ Calculates the area under the curve specified by the `x` vector and `y` vector using the trapezoidal rule. """ function area_under_curve(x, y) - @assert length(x) == length(y) + length(x) == length(y) || throw(ArgumentError("Length of inputs must match.")) + length(x) == 0 && throw(ArgumentError("Inputs must be nonempty.")) auc = zero(middle(one(eltype(x)), one(eltype(y)))) perms = sortperm(x) sorted_x = view(x, perms) @@ -35,7 +36,8 @@ Calculates the area under the curve specified by the `x` vector and `y` vector for a unit square, using the trapezoidal rule. """ function area_under_curve_unit_square(x, y) - @assert length(x) == length(y) + length(x) == length(y) || throw(ArgumentError("Length of inputs must match.")) + length(x) == 0 && throw(ArgumentError("Inputs must be nonempty.")) kept = [(i, j) for (i, j) in zip(x, y) if !(ismissing(i) || ismissing(j)) && (0 <= i <= 1 && 0 <= j <= 1)] diff --git a/test/learn.jl b/test/learn.jl index 1b926ab..f495657 100644 --- a/test/learn.jl +++ b/test/learn.jl @@ -60,7 +60,8 @@ end "test_set_evaluation/time_in_seconds_per_epoch", "test_set_evaluation/gc_time_in_seconds_per_epoch", "test_set_evaluation/allocations_per_epoch", - "test_set_evaluation/memory_in_mb_per_epoch"] + "test_set_evaluation/memory_in_mb_per_epoch", + "test_set_evaluation/metrics/accuracy_per_epoch"] @test length(logger.logged[key]) == limit end @test length(logger.logged["test_set_evaluation/metrics_per_epoch"]) == limit diff --git a/test/utilities.jl b/test/utilities.jl index 15ae555..2f5cd55 100644 --- a/test/utilities.jl +++ b/test/utilities.jl @@ -7,7 +7,8 @@ end @testset "`Lighthouse.area_under_curve`" begin - @test_throws AssertionError Lighthouse.area_under_curve([0, 1, 2], [0, 1]) + @test_throws ArgumentError Lighthouse.area_under_curve([0, 1, 2], [0, 1]) + @test_throws ArgumentError Lighthouse.area_under_curve([], []) @test isapprox(Lighthouse.area_under_curve(collect(0:0.01:1), collect(0:0.01:1)), 0.5; atol=0.01) @test isapprox(Lighthouse.area_under_curve(collect(0:0.01:(2π)), sin.(0:0.01:(2π))), @@ -15,7 +16,8 @@ end end @testset "`Lighthouse.area_under_curve_unit_square`" begin - @test_throws AssertionError Lighthouse.area_under_curve_unit_square([0, 1, 2], [0, 1]) + @test_throws ArgumentError Lighthouse.area_under_curve_unit_square([0, 1, 2], [0, 1]) + @test_throws ArgumentError Lighthouse.area_under_curve_unit_square([], []) @test isapprox(Lighthouse.area_under_curve_unit_square(collect(0:0.01:1), collect(0:0.01:1)), 0.5; atol=0.01)