wip docs and edge cases

.github/workflows/push_preview_cleanup.yml

@@ -0,0 +1,27 @@
+# from https://juliadocs.github.io/Documenter.jl/dev/man/hosting/#gh-pages-Branch
+name: Doc Preview Cleanup
+on:
+  pull_request:
+    types: [closed]
+jobs:
+  doc-preview-cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout gh-pages branch
+        uses: actions/checkout@v2
+        with:
+          ref: gh-pages
+
+      - name: Delete preview and history
+        run: |
+            git config user.name "Documenter.jl"
+            git config user.email "documenter@juliadocs.github.io"
+            git rm -rf "previews/PR$PRNUM"
+            git commit -m "delete preview"
+            git branch gh-pages-new $(echo "delete history" | git commit-tree HEAD^{tree})
+        env:
+            PRNUM: ${{ github.event.number }}
+
+      - name: Push changes
+        run: |
+            git push --force origin gh-pages-new:gh-pages

docs/make.jl

@@ -3,7 +3,9 @@ using Documenter
 
 makedocs(; modules=[Lighthouse], sitename="Lighthouse",
          authors="Beacon Biosignals and other contributors",
-         pages=["API Documentation" => "index.md"])
+         pages=["API Documentation" => "index.md",
+                "Terminology" => "terminology.md",
+                "Evaluation metrics" => "evaluation_metrics.md"])
 
 deploydocs(repo="github.com/beacon-biosignals/Lighthouse.jl.git",
-           devbranch="main")
+           devbranch="main", push_preview=true)

docs/src/evaluation_metrics.md

@@ -0,0 +1,28 @@
+# Evaluation metrics
+
+Lighthouse automatically generates a suite of evaluation metrics.
+Here, we briefly describe these. This page uses terms defined in [Terminology](@ref),
+so see that page for any unfamiliar words.
+
+## Confusion matrices
+
+Lighthouse plots confusion matrices, which are simple tables
+showing the empirical distribution of predicted class (the rows)
+versus the elected class (the columns). These come in two variants: 
+
+* row-normalized: this means each row has been normalized to sum to 1. Thus, the row-normalized confusion matrix shows the empirical distribution of elected classes for a given predicted class. E.g. the first row of the row-normalized confusion matrix shows the empirical probabilities of the elected classes for a sample which was predicted to be in the first class.
+* column-normalized: this means each column has been normalized to sum to 1. Thus, the column-normalized confusion matrix shows the empirical distribution of predicted classes for a given elected class. E.g. the first column of the column-normalized confusion matrix shows the empirical probabilities of the predicted classes for a sample which was elected to be in the first class.
+
+[insert example plot]
+
+## Inter-rater reliability
+
+## ROC curves
+
+## PR curves (precision-recall curves)
+
+## PR-gain curves (precision-recall-gain curves)
+
+## Prediction-reliability calibration
+
+##

docs/src/terminology.md

@@ -0,0 +1,9 @@
+# Terminology
+
+* _sample_: a piece of data to be classified by the model, or a labelled piece of training/test/validation data.
+* _classes_: the set of possible class labels which the model attempts to predict.
+* _voters_: the individual sources of labelled data, such as human labellers. Each voter may supply a "vote" for a class label for a sample.
+* _votes_: the matrix of votes corresponding to a set of data, whose rows correspond to the index of a sample in a set of data, whose columns correspond to voters, and whose values are the indices of class labels (i.e. numbers in `1:length(classes)`). E.g. if 2 voters have voted on ten samples, then `votes` is a 10 by 2 matrix of integers. If a voter has not voted on a particular sample, any value outside `1:length(classes)` may be supplied to indicate this.
+* _elected class_: the class elected by the voters. By default in [`learn!`](@ref),
+    the elected class is chosen by a simple majority of the votes with ties broken randomly.
+* _predicted class_: the class predicted by the model for a given input.

src/Lighthouse.jl

@@ -9,8 +9,10 @@ using StatsPlots.Plots.PlotMeasures
 using Printf
 
 # Set up plotting backend for Plots.jl (GR)
-gr()
-GR.inline("png")
+function __init__()
+    gr()
+    GR.inline("png")
+end
 
 include("utilities.jl")
 export majority

src/learn.jl

@@ -19,6 +19,11 @@ struct LearnLogger
     logged::Dict{String,Vector{Any}}
 end
 
+"""
+    LearnLogger(path, run_name; kwargs...)
+
+Creates a `LearnLogger` which stores tensorboard logs at `path` under the subdirectory `run_name`.
+"""
 function LearnLogger(path, run_name; kwargs...)
     tensorboard_logger = TBLogger(joinpath(path, run_name); kwargs...)
     return LearnLogger(path, tensorboard_logger, Dict{String,Any}())
@@ -124,16 +129,14 @@ The following quantities are logged to `logger`:
 
 Where...
 
-- `model` is a model that outputs soft labels when called on a batch of `batches`,
-`model(batch)`.
+- `model` is a model that outputs soft labels when called on a batch of `batches`, `model(batch)`.
 
 - `predicted_soft_labels` is a matrix whose columns correspond to classes and
-whose rows correspond to samples in batches, and which is filled in with soft-label
-predictions.
+    whose rows correspond to samples in batches, and which is filled in with soft-label predictions.
 
 - `batches` is an iterable of batches, where each element of
-the iterable takes the form `(batch, votes_locations)`. Internally, `batch` is
-passed to [`loss_and_prediction`](@ref) as `loss_and_prediction(model, batch...)`.
+    the iterable takes the form `(batch, votes_locations)`. Internally, `batch` is
+    passed to [`loss_and_prediction`](@ref) as `loss_and_prediction(model, batch...)`.
 
  """
 function predict!(model::AbstractClassifier, predicted_soft_labels::AbstractMatrix, batches,
@@ -180,27 +183,27 @@ The following quantities are logged to `logger`:
 Where...
 
 - `predicted_soft_labels` is a matrix of soft labels whose columns correspond to
-classes and whose rows correspond to samples in the evaluation set.
+    classes and whose rows correspond to samples in the evaluation set.
 
 - `predicted_hard_labels` is a vector of hard labels where the `i`th element
-is the hard label predicted by the model for sample `i` in the evaulation set.
+    is the hard label predicted by the model for sample `i` in the evaulation set.
 
 - `elected_hard_labels` is a vector of hard labels where the `i`th element
-is the hard label elected as "ground truth" for sample `i` in the evaulation set.
+    is the hard label elected as "ground truth" for sample `i` in the evaulation set.
 
 - `thresholds` are the range of thresholds used by metrics (e.g. PR curves) that
-are calculated on the `predicted_soft_labels` for a range of thresholds.
+    are calculated on the `predicted_soft_labels` for a range of thresholds.
 
 - `votes` is a matrix of hard labels whose columns correspond to voters and whose
-rows correspond to the samples in the test set that have been voted on. If
-`votes[sample, voter]` is not a valid hard label for `model`, then `voter` will
-simply be considered to have not assigned a hard label to `sample`.
+    rows correspond to the samples in the test set that have been voted on. If
+    `votes[sample, voter]` is not a valid hard label for `model`, then `voter` will
+    simply be considered to have not assigned a hard label to `sample`.
 
 - `optimal_threshold_class` is the class index (`1` or `2`) for which to calculate
-an optimal threshold for converting the `predicted_soft_labels` to
-`predicted_hard_labels`. If present, the input `predicted_hard_labels` will be
-ignored and new `predicted_hard_labels` will be recalculated from the new threshold.
-This is only a valid parameter when `length(classes) == 2`
+    an optimal threshold for converting the `predicted_soft_labels` to
+    `predicted_hard_labels`. If present, the input `predicted_hard_labels` will be
+    ignored and new `predicted_hard_labels` will be recalculated from the new threshold.
+    This is only a valid parameter when `length(classes) == 2`
 """
 function evaluate!(predicted_hard_labels::AbstractVector,
                    predicted_soft_labels::AbstractMatrix,
@@ -217,6 +220,7 @@ function evaluate!(predicted_hard_labels::AbstractVector,
                                                   votes=votes,
                                                   optimal_threshold_class=optimal_threshold_class)
         log_plot!(logger, logger_prefix * "/metrics" * logger_suffix, plot, plot_data)
+        log_value!(logger, logger_prefix * "/metrics/accuracy" * logger_suffix, accuracy(plot_data["confusion_matrix"]))
         if haskey(plot_data, "spearman_correlation")
             log_value!(logger, logger_prefix * "/spearman_correlation" * logger_suffix,
                        plot_data["spearman_correlation"].ρ)
@@ -250,22 +254,26 @@ function plot_roc_curves(per_class_roc_curves, per_class_roc_aucs, class_labels;
 end
 
 function plot_reliability_calibration_curves(per_class_reliability_calibration_curves,
-                                             per_class_reliability_calibration_scores,
-                                             class_labels; legend=:bottomright)
+    per_class_reliability_calibration_scores,
+    class_labels; legend=:bottomright)
+
+    eltype(per_class_reliability_calibration_scores) >: Missing && return plot()
+
     calibration_score_labels = [@sprintf("%s (MSE: %.3f)", class,
-                                         per_class_reliability_calibration_scores[i])
-                                for (i, class) in enumerate(class_labels)]
+        per_class_reliability_calibration_scores[i])
+        for (i, class) in enumerate(class_labels)]
     plot(per_class_reliability_calibration_curves; labels=calibration_score_labels,
-         title="Prediction reliability calibration", xlabel="Predicted probability bin",
-         xlims=(0, 1), ylabel="Fraction of positives", ylims=(0, 1), markershape=:circle,
-         markersize=2, linewidth=1, markerstrokewidth=0, legendfontsize=1, legend=legend,
-         framestyle=:box)
+        title="Prediction reliability calibration", xlabel="Predicted probability bin",
+        xlims=(0, 1), ylabel="Fraction of positives", ylims=(0, 1), markershape=:circle,
+        markersize=2, linewidth=1, markerstrokewidth=0, legendfontsize=1, legend=legend,
+        framestyle=:box)
     #TODO: mean predicted value histogram underneath?? Maybe important...
     # https://scikit-learn.org/stable/modules/calibration.html
     plot!([0, 1], [0, 1]; linecolor=:black, linestyle=:dash, label="Ideal")
     return xticks!(0:0.2:1)
 end
 
+
 function plot_binary_discrimination_calibration_curves(calibration_curve, calibration_score,
                                                        per_expert_calibration_curves,
                                                        per_expert_calibration_scores,
@@ -453,11 +461,9 @@ for class `i`.
 Where...
 
 - `predicted_hard_labels` is a vector of hard labels where the `i`th element
-is the hard label predicted by the model for sample `i` in the evaulation set.
-
+    is the hard label predicted by the model for sample `i` in the evaulation set.
 - `elected_hard_labels` is a vector of hard labels where the `i`th element
 is the hard label elected as "ground truth" for sample `i` in the evaulation set.
-
 - `class_count` is the number of possible classes.
 
 """
@@ -485,9 +491,9 @@ IRA kappa calculated for class `i`.
 Where...
 
 - `votes` is a matrix of hard labels whose columns correspond to voters and whose
-rows correspond to the samples in the test set that have been voted on. If
-`votes[sample, voter]` is not a valid hard label for `model`, then `voter` will
-simply be considered to have not assigned a hard label to `sample`.
+    rows correspond to the samples in the test set that have been voted on. If
+    `votes[sample, voter]` is not a valid hard label for `model`, then `voter` will
+    simply be considered to have not assigned a hard label to `sample`.
 
 - `classes` all possible classes voted on.
 
@@ -552,16 +558,14 @@ Only valid for binary classification problems (i.e., `length(classes) == 2`)
 Where...
 
 - `predicted_soft_labels` is a matrix of soft labels whose columns correspond to
-the two classes and whose rows correspond to the samples in the test set that have been
-classified. For a given sample, the two class column values must sum to 1 (i.e.,
-softmax has been applied to the classification output).
-
+    the two classes and whose rows correspond to the samples in the test set that have been
+    classified. For a given sample, the two class column values must sum to 1 (i.e.,
+    softmax has been applied to the classification output).
 - `votes` is a matrix of hard labels whose columns correspond to voters and whose
-rows correspond to the samples in the test set that have been voted on. If
+    rows correspond to the samples in the test set that have been voted on. If
 `votes[sample, voter]` is not a valid hard label for `model`, then `voter` will
-simply be considered to have not assigned a hard label to `sample`. May contain
-a single voter (i.e., a single column).
-
+    simply be considered to have not assigned a hard label to `sample`. May contain
+    a single voter (i.e., a single column).
 - `classes` are the two classes voted on.
 """
 function _calculate_spearman_correlation(predicted_soft_labels, votes, classes)
@@ -678,19 +682,19 @@ is the hard label elected as "ground truth" for sample `i` in the evaulation set
 are calculated on the `predicted_soft_labels` for a range of thresholds.
 
 - `votes` is a matrix of hard labels whose columns correspond to voters and whose
-rows correspond to the samples in the test set that have been voted on. If
-`votes[sample, voter]` is not a valid hard label for `model`, then `voter` will
-simply be considered to have not assigned a hard label to `sample`.
+    rows correspond to the samples in the test set that have been voted on. If
+    `votes[sample, voter]` is not a valid hard label for `model`, then `voter` will
+    simply be considered to have not assigned a hard label to `sample`.
 
 - `strata` is a vector of sets of (arbitrarily typed) groups/strata for each sample
-in the evaluation set, or `nothing`. If not `nothing`, per-class and multiclass
-kappas will also be calculated per group/stratum.
+    in the evaluation set, or `nothing`. If not `nothing`, per-class and multiclass
+    kappas will also be calculated per group/stratum.
 
 - `optimal_threshold_class` is the class index (`1` or `2`) for which to calculate
-an optimal threshold for converting the `predicted_soft_labels` to
-`predicted_hard_labels`. If present, the input `predicted_hard_labels` will be
-ignored and new `predicted_hard_labels` will be recalculated from the new threshold.
-This is only a valid parameter when `length(classes) == 2`
+    an optimal threshold for converting the `predicted_soft_labels` to
+    `predicted_hard_labels`. If present, the input `predicted_hard_labels` will be
+    ignored and new `predicted_hard_labels` will be recalculated from the new threshold.
+    This is only a valid parameter when `length(classes) == 2`
 """
 function evaluation_metrics_plot(predicted_hard_labels::AbstractVector,
                                  predicted_soft_labels::AbstractMatrix,
@@ -840,7 +844,7 @@ end
     learn!(model::AbstractClassifier, logger,
            get_train_batches, get_test_batches, votes,
            elected=majority.(eachrow(votes), (1:length(classes(model)),));
-           epoch_limit=100, post_epoch_callback=(_ -> nothing),
+           epoch_limit=100, post_epoch_callback=(current_epoch -> nothing),
            optimal_threshold_class::Union{Nothing,Integer}=nothing)
 
 Return `model` after optimizing its parameters across multiple epochs of
@@ -875,34 +879,34 @@ of logged values, `\$resource` takes the values of the field names of
 Where...
 
 - `get_train_batches` is a zero-argument function that returns an iterable of
-training set batches. Internally, `learn!` uses this function when it calls
-`train!(model, get_train_batches(), logger)`.
+    training set batches. Internally, `learn!` uses this function when it calls
+    `train!(model, get_train_batches(), logger)`.
 
 - `get_test_batches` is a zero-argument function that returns an iterable
-of test set batches used during the current epoch's test phase. Each element of
-the iterable takes the form `(batch, votes_locations)`. Internally, `batch` is
-passed to [`loss_and_prediction`](@ref) as `loss_and_prediction(model, batch...)`,
-and `votes_locations[i]` is expected to yield the row index of `votes` that
-corresponds to the `i`th sample in `batch`.
+    of test set batches used during the current epoch's test phase. Each element of
+    the iterable takes the form `(batch, votes_locations)`. Internally, `batch` is
+    passed to [`loss_and_prediction`](@ref) as `loss_and_prediction(model, batch...)`,
+    and `votes_locations[i]` is expected to yield the row index of `votes` that
+    corresponds to the `i`th sample in `batch`.
 
 - `votes` is a matrix of hard labels whose columns correspond to voters and whose
-rows correspond to the samples in the test set that have been voted on. If
-`votes[sample, voter]` is not a valid hard label for `model`, then `voter` will
-simply be considered to have not assigned a hard label to `sample`.
+    rows correspond to the samples in the test set that have been voted on. If
+    `votes[sample, voter]` is not a valid hard label for `model`, then `voter` will
+    simply be considered to have not assigned a hard label to `sample`.
 
 - `elected` is a vector of hard labels where the `i`th element is the hard label
-elected as "ground truth" out of `votes[i, :]`.
+    elected as "ground truth" out of `votes[i, :]`.
 
 - `optimal_threshold_class` is the class index (`1` or `2`) for which to calculate
-an optimal threshold for converting `predicted_soft_labels` to `predicted_hard_labels`.
-This is only a valid parameter when `length(classes) == 2`. If `optimal_threshold_class`
-is present, test set evaluation will be based on predicted hard labels calculated
-with this threshold; if `optimal_threshold_class` is `nothing`, predicted hard labels
-will be calculated via `onecold(classifier, soft_label)`.
+    an optimal threshold for converting `predicted_soft_labels` to `predicted_hard_labels`.
+    This is only a valid parameter when `length(classes) == 2`. If `optimal_threshold_class`
+    is present, test set evaluation will be based on predicted hard labels calculated
+    with this threshold; if `optimal_threshold_class` is `nothing`, predicted hard labels
+    will be calculated via `onecold(classifier, soft_label)`.
 """
 function learn!(model::AbstractClassifier, logger, get_train_batches, get_test_batches,
                 votes, elected=majority.(eachrow(votes), (1:length(classes(model)),));
-                epoch_limit=100, post_epoch_callback=(_ -> nothing),
+                epoch_limit=100, post_epoch_callback=(current_epoch -> nothing),
                 optimal_threshold_class::Union{Nothing,Integer}=nothing)
     # NOTE `votes` is currently unused except to construct `elected` by default,
     # but will be necessary for calculating multirater metrics e.g. Fleiss' kappa

src/utilities.jl

@@ -16,7 +16,8 @@ Calculates the area under the curve specified by the `x` vector and `y` vector
 using the trapezoidal rule.
 """
 function area_under_curve(x, y)
-    @assert length(x) == length(y)
+    length(x) == length(y) || throw(ArgumentError("Length of inputs must match."))
+    length(x) == 0 && throw(ArgumentError("Inputs must be nonempty."))
     auc = zero(middle(one(eltype(x)), one(eltype(y))))
     perms = sortperm(x)
     sorted_x = view(x, perms)
@@ -35,7 +36,8 @@ Calculates the area under the curve specified by the `x` vector and `y` vector
 for a unit square, using the trapezoidal rule.
 """
 function area_under_curve_unit_square(x, y)
-    @assert length(x) == length(y)
+    length(x) == length(y) || throw(ArgumentError("Length of inputs must match."))
+    length(x) == 0 && throw(ArgumentError("Inputs must be nonempty."))
     kept = [(i, j)
             for (i, j) in zip(x, y)
             if !(ismissing(i) || ismissing(j)) && (0 <= i <= 1 && 0 <= j <= 1)]