From e1cad06afb30f10f7a1e482f2ca56d47b3ad6abe Mon Sep 17 00:00:00 2001 From: matthieugomez Date: Sat, 17 Oct 2020 12:03:56 -0700 Subject: [PATCH 1/8] simplify progressbar --- Project.toml | 3 +- .../FixedEffectSolverCPU.jl | 12 ++- src/FixedEffects.jl | 6 +- src/{ => utils}/lsmr.jl | 0 src/utils/progressbar.jl | 74 +++++++++++++++++++ 5 files changed, 88 insertions(+), 7 deletions(-) rename src/{ => utils}/lsmr.jl (100%) create mode 100644 src/utils/progressbar.jl diff --git a/Project.toml b/Project.toml index 902c28d..4aeaeda 100644 --- a/Project.toml +++ b/Project.toml @@ -4,12 +4,11 @@ version = "2.0.0" [deps] LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" -ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" Requires = "ae029012-a4dd-5104-9daa-d747884805df" StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" [compat] -ProgressMeter = "1" Requires = "1" StatsBase = "0.33" julia = "1.3" diff --git a/src/FixedEffectSolvers/FixedEffectSolverCPU.jl b/src/FixedEffectSolvers/FixedEffectSolverCPU.jl index dadf419..6f7bf09 100644 --- a/src/FixedEffectSolvers/FixedEffectSolverCPU.jl +++ b/src/FixedEffectSolvers/FixedEffectSolverCPU.jl @@ -166,18 +166,24 @@ end function solve_residuals!(X::AbstractMatrix, feM::FixedEffects.FixedEffectSolverCPU; progress_bar = true, kwargs...) iterations = Int[] convergeds = Bool[] - progress_bar = ifelse(size(X, 2) <= 4, false, progress_bar) + io = stdout if progress_bar - p = Progress(size(X, 2); dt = 1, desc = "Demeaning Variables...", color = :normal) + bar = MiniProgressBar(header = "Demeaning:", color = Base.info_color(), percentage = false, always_reprint=true) + bar.max = size(X, 2) + showprogress(io, bar) end for j in 1:size(X, 2) _, iteration, converged = solve_residuals!(view(X, :, j), feM; kwargs...) push!(iterations, iteration) push!(convergeds, converged) if progress_bar - next!(p) + bar.current = j + showprogress(io, bar) end end + if progress_bar + print() + end return X, iterations, convergeds end diff --git a/src/FixedEffects.jl b/src/FixedEffects.jl index 3ad1917..100bac4 100644 --- a/src/FixedEffects.jl +++ b/src/FixedEffects.jl @@ -8,14 +8,16 @@ module FixedEffects using LinearAlgebra using StatsBase using Requires -using ProgressMeter +using Printf ############################################################################## ## ## Load files ## ############################################################################## -include("lsmr.jl") +include("utils/lsmr.jl") +include("utils/progressbar.jl") + include("FixedEffect.jl") include("AbstractFixedEffectSolver.jl") include("FixedEffectSolvers/FixedEffectLinearMap.jl") diff --git a/src/lsmr.jl b/src/utils/lsmr.jl similarity index 100% rename from src/lsmr.jl rename to src/utils/lsmr.jl diff --git a/src/utils/progressbar.jl b/src/utils/progressbar.jl new file mode 100644 index 0000000..ab7e10d --- /dev/null +++ b/src/utils/progressbar.jl @@ -0,0 +1,74 @@ +Base.@kwdef mutable struct MiniProgressBar + max::Int = 1.0 + header::String = "" + color::Symbol = :white + width::Int = 40 + current::Int = 0.0 + prev::Int = 0.0 + has_shown::Bool = false + time_shown::Float64 = 0.0 + percentage::Bool = true + always_reprint::Bool = false + indent::Int = 4 +end + +const NONINTERACTIVE_TIME_GRANULARITY = Ref(2.0) +const PROGRESS_BAR_PERCENTAGE_GRANULARITY = Ref(0.1) + +function showprogress(io::IO, p::MiniProgressBar) + if p.max == 0 + perc = 0.0 + prev_perc = 0.0 + else + perc = p.current / p.max * 100 + prev_perc = p.prev / p.max * 100 + end + # Bail early if we are not updating the progress bar, + # Saves printing to the terminal + if !p.always_reprint && p.has_shown && !((perc - prev_perc) > PROGRESS_BAR_PERCENTAGE_GRANULARITY[]) + return + end + if !isinteractive() + t = time() + if p.has_shown && (t - p.time_shown) < NONINTERACTIVE_TIME_GRANULARITY[] + return + end + p.time_shown = t + end + p.prev = p.current + p.has_shown = true + n_filled = ceil(Int, p.width * perc / 100) + n_left = p.width - n_filled + print(io, " "^p.indent) + printstyled(io, p.header, color=p.color, bold=true) + print(io, " [") + print(io, "="^n_filled, ">") + print(io, " "^n_left, "] ", ) + if p.percentage + @printf io "%2.1f %%" perc + else + print(io, p.current, "/", p.max) + end + print(io, "\r") +end + +# Useful when writing a progress bar in the bottom +# makes the bottom progress bar not flicker +# prog = MiniProgressBar(...) +# prog.end = n +# for progress in 1:n +# print_progree_bottom(io) +# println("stuff") +# prog.current = progress +# showproress(io, prog) +# end +# +function print_progress_bottom(io::IO) + print(io, "\e[S") + ansi_cleartoend = "\e[0J" + ansi_movecol1 = "\e[1G" + ansi_moveup(n::Int) = string("\e[", n, "A") + print(io, ansi_moveup(1)) + print(io, ansi_movecol1) + print(io, ansi_cleartoend) +end \ No newline at end of file From c8ae7454ad011b4024b1d7d0028e451ccb0cd9a7 Mon Sep 17 00:00:00 2001 From: matthieugomez Date: Sat, 17 Oct 2020 13:04:29 -0700 Subject: [PATCH 2/8] Update FixedEffectSolverCPU.jl --- src/FixedEffectSolvers/FixedEffectSolverCPU.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FixedEffectSolvers/FixedEffectSolverCPU.jl b/src/FixedEffectSolvers/FixedEffectSolverCPU.jl index 6f7bf09..97647a8 100644 --- a/src/FixedEffectSolvers/FixedEffectSolverCPU.jl +++ b/src/FixedEffectSolvers/FixedEffectSolverCPU.jl @@ -182,7 +182,7 @@ function solve_residuals!(X::AbstractMatrix, feM::FixedEffects.FixedEffectSolver end end if progress_bar - print() + print(io) end return X, iterations, convergeds end From 51421b6d34cf8bebdd3c870a9db61d002fcb1b18 Mon Sep 17 00:00:00 2001 From: matthieugomez Date: Sat, 17 Oct 2020 13:08:24 -0700 Subject: [PATCH 3/8] Update progressbar.jl --- src/utils/progressbar.jl | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/utils/progressbar.jl b/src/utils/progressbar.jl index ab7e10d..22e7d1d 100644 --- a/src/utils/progressbar.jl +++ b/src/utils/progressbar.jl @@ -50,25 +50,4 @@ function showprogress(io::IO, p::MiniProgressBar) print(io, p.current, "/", p.max) end print(io, "\r") -end - -# Useful when writing a progress bar in the bottom -# makes the bottom progress bar not flicker -# prog = MiniProgressBar(...) -# prog.end = n -# for progress in 1:n -# print_progree_bottom(io) -# println("stuff") -# prog.current = progress -# showproress(io, prog) -# end -# -function print_progress_bottom(io::IO) - print(io, "\e[S") - ansi_cleartoend = "\e[0J" - ansi_movecol1 = "\e[1G" - ansi_moveup(n::Int) = string("\e[", n, "A") - print(io, ansi_moveup(1)) - print(io, ansi_movecol1) - print(io, ansi_cleartoend) end \ No newline at end of file From 1db684c2400e065388c439a880369f242d2fce8c Mon Sep 17 00:00:00 2001 From: matthieugomez Date: Sat, 17 Oct 2020 15:04:39 -0700 Subject: [PATCH 4/8] Update FixedEffectSolverCPU.jl --- src/FixedEffectSolvers/FixedEffectSolverCPU.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FixedEffectSolvers/FixedEffectSolverCPU.jl b/src/FixedEffectSolvers/FixedEffectSolverCPU.jl index 97647a8..925caa0 100644 --- a/src/FixedEffectSolvers/FixedEffectSolverCPU.jl +++ b/src/FixedEffectSolvers/FixedEffectSolverCPU.jl @@ -168,7 +168,7 @@ function solve_residuals!(X::AbstractMatrix, feM::FixedEffects.FixedEffectSolver convergeds = Bool[] io = stdout if progress_bar - bar = MiniProgressBar(header = "Demeaning:", color = Base.info_color(), percentage = false, always_reprint=true) + bar = MiniProgressBar(header = "Demean Variables:", color = Base.info_color(), percentage = false, always_reprint=true) bar.max = size(X, 2) showprogress(io, bar) end From eec1dcc916d24f5998d56c2fec575bc3da87e7c0 Mon Sep 17 00:00:00 2001 From: matthieugomez Date: Sun, 15 Nov 2020 13:47:42 -0800 Subject: [PATCH 5/8] correct weights --- src/FixedEffectSolvers/FixedEffectSolverGPU.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/FixedEffectSolvers/FixedEffectSolverGPU.jl b/src/FixedEffectSolvers/FixedEffectSolverGPU.jl index bff22f7..4ad520a 100644 --- a/src/FixedEffectSolvers/FixedEffectSolverGPU.jl +++ b/src/FixedEffectSolvers/FixedEffectSolverGPU.jl @@ -127,12 +127,12 @@ function AbstractFixedEffectSolver{T}(fes::Vector{<:FixedEffect}, weights::Abstr h = FixedEffectCoefficients([cuzeros(T, fe.n) for fe in fes]) hbar = FixedEffectCoefficients([cuzeros(T, fe.n) for fe in fes]) tmp = zeros(T, length(weights)) - update_weights!(FixedEffectSolverGPU{T}(m, weights, b, r, x, v, h, hbar, tmp, fes), weights) + update_weights!(FixedEffectSolverGPU{T}(m, cuzeros(T, length(weights)), b, r, x, v, h, hbar, tmp, fes), weights) end function update_weights!(feM::FixedEffectSolverGPU{T}, weights::AbstractWeights) where {T} - weights = cu(T, collect(weights)) + weights = cu(T, weights) for (scale, fe) in zip(feM.m.scales, feM.m.fes) scale!(scale, fe.refs, fe.interaction, weights, fe.m.nthreads) end From b9ec8cb5e085bbec105c814bad7121cfc1cb38eb Mon Sep 17 00:00:00 2001 From: matthieugomez Date: Sun, 15 Nov 2020 13:54:51 -0800 Subject: [PATCH 6/8] Update Project.toml --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 4aeaeda..bc560d5 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "FixedEffects" uuid = "c8885935-8500-56a7-9867-7708b20db0eb" -version = "2.0.0" +version = "2.0.1" [deps] LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" From e6ef6c77e68d888e8dbea0ab22c081c95935dd5d Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 4 Feb 2021 17:05:47 +0100 Subject: [PATCH 7/8] Fixing `nthreads` Fix `nthreads` before the loop in `update_weights!` when run on GPU --- src/FixedEffectSolvers/FixedEffectSolverGPU.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/FixedEffectSolvers/FixedEffectSolverGPU.jl b/src/FixedEffectSolvers/FixedEffectSolverGPU.jl index 4ad520a..8503a3e 100644 --- a/src/FixedEffectSolvers/FixedEffectSolverGPU.jl +++ b/src/FixedEffectSolvers/FixedEffectSolverGPU.jl @@ -133,11 +133,12 @@ end function update_weights!(feM::FixedEffectSolverGPU{T}, weights::AbstractWeights) where {T} weights = cu(T, weights) + nthreads = feM.m.nthreads for (scale, fe) in zip(feM.m.scales, feM.m.fes) - scale!(scale, fe.refs, fe.interaction, weights, fe.m.nthreads) + scale!(scale, fe.refs, fe.interaction, weights, nthreads) end for (cache, scale, fe) in zip(feM.m.caches, feM.m.scales, feM.m.fes) - cache!(cache, fe.refs, fe.interaction, weights, scale, fe.m.nthreads) + cache!(cache, fe.refs, fe.interaction, weights, scale, nthreads) end feM.weights = weights return feM @@ -222,4 +223,4 @@ function solve_coefficients!(r::AbstractVector, feM::FixedEffectSolverGPU{T}; to end x = Vector{eltype(r)}[collect(x) for x in feM.x.x] full(normalize!(x, feM.fes; tol = tol, maxiter = maxiter), feM.fes), div(ch.mvps, 2), ch.isconverged -end \ No newline at end of file +end From 96db85996a5eb9adf25d9abd2612a43e7f388e39 Mon Sep 17 00:00:00 2001 From: Daniel Winkler Date: Fri, 12 Feb 2021 18:42:31 +0100 Subject: [PATCH 8/8] removed duplicate weights creation --- src/FixedEffectSolvers/FixedEffectSolverGPU.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/FixedEffectSolvers/FixedEffectSolverGPU.jl b/src/FixedEffectSolvers/FixedEffectSolverGPU.jl index d9e0d93..8503a3e 100644 --- a/src/FixedEffectSolvers/FixedEffectSolverGPU.jl +++ b/src/FixedEffectSolvers/FixedEffectSolverGPU.jl @@ -132,7 +132,6 @@ end function update_weights!(feM::FixedEffectSolverGPU{T}, weights::AbstractWeights) where {T} - weights = cu(T, collect(weights)) weights = cu(T, weights) nthreads = feM.m.nthreads for (scale, fe) in zip(feM.m.scales, feM.m.fes)