From 94fdcebab8d37ed7409e951d6a6fc44079d0bee9 Mon Sep 17 00:00:00 2001 From: Carlo Lucibello Date: Sun, 31 Oct 2021 19:16:39 +0100 Subject: [PATCH 1/4] test with sparse cuda arrays --- src/GNNGraphs/GNNGraphs.jl | 1 + src/GNNGraphs/convert.jl | 30 +++++++++++++++++++++++++++++- src/GNNGraphs/gnngraph.jl | 4 ++-- src/GNNGraphs/query.jl | 7 +------ src/msgpass.jl | 8 ++++---- 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/GNNGraphs/GNNGraphs.jl b/src/GNNGraphs/GNNGraphs.jl index 51e8891c6..7bffa6686 100644 --- a/src/GNNGraphs/GNNGraphs.jl +++ b/src/GNNGraphs/GNNGraphs.jl @@ -3,6 +3,7 @@ module GNNGraphs using SparseArrays using Functors: @functor using CUDA +using CUDA.CUSPARSE import Graphs using Graphs: AbstractGraph, outneighbors, inneighbors, adjacency_matrix, degree import Flux diff --git a/src/GNNGraphs/convert.jl b/src/GNNGraphs/convert.jl index 09f0de586..2036432d1 100644 --- a/src/GNNGraphs/convert.jl +++ b/src/GNNGraphs/convert.jl @@ -137,11 +137,39 @@ function to_sparse(coo::COO_T, T::DataType=Int; dir=:out, num_nodes=nothing) s, t, eweight = coo eweight = isnothing(eweight) ? fill!(similar(s, T), 1) : eweight num_nodes = isnothing(num_nodes) ? max(maximum(s), maximum(t)) : num_nodes - A = sparse(s, t, eweight, num_nodes, num_nodes) + A = _sparse(s, t, eweight, num_nodes, num_nodes) num_edges = length(s) return A, num_nodes, num_edges end +_sparse(s, t, eweight, n, m) = sparse(s, t, eweight, n, m) + +function _sparse(I::CuVector, J::CuVector, V::CuVector, m, n) + spcoo = CuSparseMatrixCOO{Float32, Int32}(Int32.(I), Int32.(J), Float32.(V), (m, n)) + return CuSparseMatrixCSR(spcoo) +end + +# function _sparse(I::CuVector, J::CuVector, V::CuVector, m, n; fmt=:csr) +# # Tv = Int32 +# spcoo = CuSparseMatrixCOO{Float32, Int32}(Int32.(I), Int32.(J), Float32.(V), (m, n)) +# if fmt == :csc +# return CuSparseMatrixCSC(spcoo) +# elseif fmt == :csr +# return CuSparseMatrixCSR(spcoo) +# elseif fmt == :coo +# return spcoo +# else +# error("Format :$fmt not available, use :csc, :csr, or :coo.") +# end +# end + + +# Workaround for https://github.com/JuliaGPU/CUDA.jl/issues/1113#issuecomment-955759875 +function Base.:*(A::CuMatrix, B::CuSparseMatrixCSR) + @assert size(A, 2) == size(B, 1) + return CuMatrix((B' * A')') +end + @non_differentiable to_coo(x...) @non_differentiable to_dense(x...) diff --git a/src/GNNGraphs/gnngraph.jl b/src/GNNGraphs/gnngraph.jl index a0bf2440f..019586f05 100644 --- a/src/GNNGraphs/gnngraph.jl +++ b/src/GNNGraphs/gnngraph.jl @@ -192,8 +192,8 @@ function GNNGraph(g::GNNGraph; ndata=g.ndata, edata=g.edata, gdata=g.gdata, grap ndata, edata, gdata) end -function Base.show(io::IO, g::GNNGraph) - println(io, "GNNGraph: +function Base.show(io::IO, g::GNNGraph{T}) where T + println(io, "GNNGraph{$T}: num_nodes = $(g.num_nodes) num_edges = $(g.num_edges) num_graphs = $(g.num_graphs)") diff --git a/src/GNNGraphs/query.jl b/src/GNNGraphs/query.jl index bb4fb6f29..5e580f284 100644 --- a/src/GNNGraphs/query.jl +++ b/src/GNNGraphs/query.jl @@ -74,12 +74,7 @@ function adjacency_list(g::GNNGraph; dir=:out) end function Graphs.adjacency_matrix(g::GNNGraph{<:COO_T}, T::DataType=Int; dir=:out) - if g.graph[1] isa CuVector - # TODO revisit after https://github.com/JuliaGPU/CUDA.jl/pull/1152 - A, n, m = to_dense(g.graph, T, num_nodes=g.num_nodes) - else - A, n, m = to_sparse(g.graph, T, num_nodes=g.num_nodes) - end + A, n, m = to_sparse(g.graph, T, num_nodes=g.num_nodes) @assert size(A) == (n, n) return dir == :out ? A : A' end diff --git a/src/msgpass.jl b/src/msgpass.jl index 1611ebe56..ece15a1d6 100644 --- a/src/msgpass.jl +++ b/src/msgpass.jl @@ -152,10 +152,10 @@ function propagate(::typeof(copyxj), g::GNNGraph, ::typeof(+), xi, xj::AbstractM return xj * A end -## avoid the fast path on gpu until we have better cuda support -function propagate(::typeof(copyxj), g::GNNGraph{<:Union{COO_T,SPARSE_T}}, ::typeof(+), xi, xj::AnyCuMatrix, e) - propagate((xi,xj,e)->copyxj(xi,xj,e), g, +, xi, xj, e) -end +# ## avoid the fast path on gpu until we have better cuda support +# function propagate(::typeof(copyxj), g::GNNGraph{<:Union{COO_T,SPARSE_T}}, ::typeof(+), xi, xj::AnyCuMatrix, e) +# propagate((xi,xj,e) -> copyxj(xi,xj,e), g, +, xi, xj, e) +# end # function propagate(::typeof(copyxj), g::GNNGraph, ::typeof(mean), xi, xj::AbstractMatrix, e) # A = adjacency_matrix(g) From 836b2c6d5e31238e39c9d3d89e90788533622c24 Mon Sep 17 00:00:00 2001 From: CarloLucibello Date: Mon, 1 Nov 2021 01:04:35 +0100 Subject: [PATCH 2/4] revamp benchmarks --- perf/Project.toml | 4 ++- perf/master_2021_11_01_arrakis.jld2 | Bin 0 -> 15926 bytes perf/perf.jl | 53 ++++++++++++++++++---------- src/GNNGraphs/generate.jl | 6 ++-- 4 files changed, 41 insertions(+), 22 deletions(-) create mode 100644 perf/master_2021_11_01_arrakis.jld2 diff --git a/perf/Project.toml b/perf/Project.toml index ddbb1be6e..825ed5a2b 100644 --- a/perf/Project.toml +++ b/perf/Project.toml @@ -1,6 +1,8 @@ [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" GraphNeuralNetworks = "cffab07f-9bc2-4db1-8861-388f63bf7694" +Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" -Graphs = "093fc24a-ae57-5d10-9952-331d41423f4d" +Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" diff --git a/perf/master_2021_11_01_arrakis.jld2 b/perf/master_2021_11_01_arrakis.jld2 new file mode 100644 index 0000000000000000000000000000000000000000..6d058a2490f0bdd96f331d83310433fcb33e2a53 GIT binary patch literal 15926 zcmeGj4OmoF_QD{JfQm|{`O_$<_=kW9BI-N@5fwEN)ZH=~U_=yVXc#aw%eFRu+R7}~ za<#JB%JL^O-K{LGO;^KKOUo^5t+jQvKPAhy@@LO^_Z)p(XK)^$Zr^^N_xqUh&bjxT zd(OG%{+;*8&deInx6o1PEVhlRDl2u^G96xrEz9jGcX<2Qs-2$7Qn$+%AD0l9AZ)R+ zNkUw5Tz^|~Qs2T-uPtYIPjRLE-+Ev{c4k)duxrACtyUp|r5R_bBoCGNiZlu1oa0-J z%g)RV7V|7ag_aD>ZBXNKlskn;Z51RU0gH#fAovr)Dk>ZvsPjS)iO5u-V10^SBSb_x z^cNH^!bFIej*(a3t*LMdOE3z+g32_9za5A`kwWD&|0XYMmLs1>WQ@zvTVNkhf~Y-8 zz>?wiIO8z3^RO3`F6A}TYlRpFQy?z0w8%@ih3dlz*ZC*Zloz_obV>--hZ*Zfy1dCr zlF)?eNgXk}m82gc80w2Z-Q#i8$aYPszMg*9IZ8d$pYmh! zwIn*ZOO7A%xu*_ud%f;*Y8OfEAbRzp%TIUJv?4m0EBVt4E4>~^k=Nko*3^#2>4~;d z&X9HzwxRl_5)~uT^D^A7Y7ym!ew813TR-%xReHu)U4A=1^!9$}*Z84#Ai9P0FLJx} zr;dK;o&3-{`=MX!hi>yj@8XBv)ek+I=sZ3UTHdKuWpU%H94>FEw%~`AncG1# zu^ZKM958=oX}Qyddt0S+Om}L>ajdTgxehP1a9|wwpn8rop2x97=QzDUKKZ>TwF@CU z5BkT)UW(kcon?dGM6b8L0CneQTUE_l6-v?`fJxpdK}SJJ+nMcC+bXY5dEkf z`;+ZKh?3%|K2(Yo6P!hESFsQYMCWn(RTtUdVS_R0De_Nmo5WM79oxb7bG!|t`iuE7Rq5}C z7%6{fHwnuL4pQ2&UJ%%Nd6r-rwX3%ujjbv#bn45f7)7n=t|shz)ZvQs^e z)4X`8r$+RP`6c}#@`q79+re?bFoWuOyvq}1|GNnOOfXYv$9~;J`u>aHaBAmopUqPA zQTOo?RL}d@SgDT4CVET4OFWL^f|81=0@Yr1~aYub+N|7^Tq595wm~k5>9A zBEB4=^Sb4Qb)|qigEnKRp68EVKXqoYt_0MN^~;Z13Ng+fKiGmQTu1e){o@rn>zhk- zwwvvnpwLUmA9+Ogr+>b`deUzq)nDizY2PHj^-{oOs`odJ=mRYC*&vBt`Mu(SD;~Jw zfh!*Pg*+hh6c1ahLNk?dz;nP3(it-l_{TUd@evuAwrH-WEoOR3yjL<$SKi z3a88vVFRnqi-U$2l&|+~`$>x1YDtk9AUudoE%j8w?ZB56mGXWN8E?ge5Ed=4)Q@o> z@bYc+APR~YjI>A)8Ug(~%Zdw%A%%gbhRSm}reMedAs-e#xJgWi-L7M&r73ZWXCS3v z)Yb||j}YjFPdE=XPwpuhU$FF$KET6jncF?1s^WZp3$tRC!fGn`P!5kKAHZ2(gl292 zI|?+E4Y|9fOV#&UMx7tNB6nF;IbMWWSs;{R5=dAv$9Ji$!&DMm$JZExa;B|g-sCp& z>A~tX$9{zPVb`ZyZ+fYsp?~83bCx zVqk+B|GC++LJ`U4R27PL{2b7-g9flMe6p5$$lr*}z$PazS?c6uSta?$K9*MbT)uX( zQ)v0ZDavZKfaeNw&SM03%pZ1-6z9c_0S6vg0x}sdwk(@313)@gA|R0vEJ9&53K7oo z3U5ulOgOGWK{vd8c&F?>rrr<-%f+~kQjb%AG^07XUa zrGbZ03}U)92i=w}I=6=gP?i#myhh^h5}e;b;+>!yELA-ve5bdB-Z%*l4mh(>8tAI8 zCpsF7McpVM=tG32Z=zUVj6YD0P?SefhrF`;t6w~Q&GU`wm~(QCg-?9Ft)uLClOKyA z>+Nfkx88-8&lR{&a|q&UYs7K&H{v*gMjV6Rh~tzs;<#EHaU5MEj-wm=z#BuQ0jvEO zpzj2{+OXOU) zarI_IrVPk;qM;Kg%ezUqmta_TiKi31h2T1ZhY5bR@R3tezv-4Pu(!aPiF2SD6XOXs zyN(D5{G@P^bQX*cXg$Rxik*)78J(64c=(OoX?+Q3xiaoZF~F{tgJ&mCk~WxM`i)B= zy3V-G^aVVCrGL|^x*ufsm6l%T4;8Mea!9FzGtr;u!eXaOSdD~$d}%ye;FuW|>;brE z#>KtMG5Cc>dw^J4so2F6AW)lftC zcnrHf_SEi1jm>_`L>bxWOpI(W?0@N3W2`5F3dUIP54hP_=a)W{s%xK<4)J-j`)e1F z4TW!ZKhgIq9P12kZo8;yqx@=q%P>Y5talfb0Je-TGW!=cpzXD|X+q%{`eu5{64|^D z9U@=_#K4x9i&&n}pc!T}ree8oiq`&MSNf*sGVSenJU34NL4dz-cK(_MOuK}O&p@U3 zRKWke=j3g&_kg15w4Z1?v~2EMjWWXLf@^X(L+Fgd$*I?WNSU_o%yIjMPSa*J5CtmR z-G>L=`_kkXZOH6B9usUEpB?9!S9OziYloXChZa!G9qpcAUA;VBi+yL;yaswxL-oZK z$DY~VW~>(6E|MlhK)ro7D{AeC_h)IT&S#D_(3|R#*tvzpJ7ZdF>t|;jlx&yhQ2rVo z%%FTnus5ESpcE2(gy0^6P3f9Cl;FOzvz9ermg>YC?tk<9*iAFF5AWViJ8wWvykkkm zJ!MUA(K7E^RM$XnsuS}%e`397VV!;TvDYc#;5rc@nEHDT7X+efD?c=I>ffUJj~Y=BK2 z=_Xrm32nZjiMIKOZN3S%Jq1TvJ(-ZEO%L0>#ROYOhy7b-h?}+bJ0j=`5m11PiU{sK z?>{rOj;o&hsDa+p0BL>a*0)zqyHC6RV0u9VY^qBN<_!MslPMjw2fJ;VYJzRTjlEZ< zIi1?fGQqay zRARg6#dY?N){RpR}|$K1{n&hSpcEs~6rgpg0IVLGTd47&?t# zPjE58y##j_H>1}um+3q_7?d-fhcf~H{Pflu>0|RZ4P$a@2*@(tx$J-kaMM^CS3*Y# z^WBF*7d+nd?*c$MUjpd<)EnFLrS(2f##X!%L*a`~{;B698)&`&FZ71yW5#sNGZDYW z%!LMGj6;(?c9=W+L^_5+J*+eoWS3fQsSa&+z|FWKjOf0*8h zn-q|9di7UBmOp5pCx)Dn8u&aisg@skD*y3!y(RXY2@~nXZ9r^Y#;nS$T$^kEHg1x& zQTDDMF{4ZKUEA!Rk?`9I<0dl#*m-u&{%%em6Jqc^I(si0p3$3FHjj>`(_28=GyxVCGZLOL z^^fJ9>>u4Gv@wbq1?;%=V~IF=hL)Cqv@!Z-G_YexVELlfG1i1O&4GoDg2ARYL_+Le zvGsEkVqir&CuR&hqmfu(Yh;i$UX}zp;RuDE$lq@%^#1Gzg`ULIRG|;x zFQgQDGJg`K&{OzhBZWSYzqe87sr= gpu m = CONV(D => D) @@ -58,11 +71,12 @@ function run_benchmarks(; c = 6, D = 100, layers = [GCNConv, GATConv], - gtypes = [:coo, :sparse, :dense], + gtypes = [:coo], ) - df = DataFrame(N=Int[], c=Float64[], layer=String[], gtype=Symbol[], - time_cpu=Any[], time_gpu=Any[]) |> allowmissing + df = DataFrame(N=Int[], c=Int[], layer=String[], gtype=Symbol[], + time_fwd_cpu=Any[], time_fwd_gpu=Any[], + time_grad_cpu=Any[], time_grad_gpu=Any[]) for gtype in gtypes for N in Ns @@ -73,31 +87,34 @@ function run_benchmarks(; N = N, c = c, gtype = gtype, - time_cpu = ismissing(res["CPU"]) ? missing : median(res["CPU"]), - time_gpu = ismissing(res["GPU"]) ? missing : median(res["GPU"]), + time_fwd_cpu = getres(res, "CPU_FWD"), + time_fwd_gpu = getres(res, "GPU_FWD"), + time_grad_cpu = getres(res, "CPU_GRAD"), + time_grad_gpu = getres(res, "GPU_GRAD"), ) push!(df, row) + println(row) end end end - df.gpu_to_cpu = ratio.(df.time_gpu, df.time_cpu) + df.grad_gpu_to_cpu = NoUnits.(df.time_grad_gpu ./ df.time_grad_cpu) sort!(df, [:layer, :N, :c, :gtype]) return df end -# df = run_benchmarks() -# for g in groupby(df, :layer); println(g, "\n"); end +df = run_benchmarks() +for g in groupby(df, :layer); println(g, "\n"); end -# @save "perf/perf_master_20210803_carlo.jld2" dfmaster=df +# @save "master_2021_11_01_arrakis.jld2" dfmaster=df ## or -# @save "perf/perf_pr.jld2" dfpr=df +# @save "pr.jld2" dfpr=df function compare(dfpr, dfmaster; on=[:N, :c, :gtype, :layer]) df = outerjoin(dfpr, dfmaster; on=on, makeunique=true, renamecols = :_pr => :_master) - df.pr_to_master_cpu = ratio.(df.time_cpu_pr, df.time_cpu_master) - df.pr_to_master_gpu = ratio.(df.time_gpu_pr, df.time_gpu_master) + df.pr_to_master_cpu = df.time_cpu_pr ./ df.time_cpu_master + df.pr_to_master_gpu = df.time_gpu_pr ./ df.time_gpu_master return df[:,[:N, :c, :gtype, :layer, :pr_to_master_cpu, :pr_to_master_gpu]] end diff --git a/src/GNNGraphs/generate.jl b/src/GNNGraphs/generate.jl index 67f5d1f1b..9f0eeb5d2 100644 --- a/src/GNNGraphs/generate.jl +++ b/src/GNNGraphs/generate.jl @@ -1,5 +1,5 @@ """ - rand_graph(n, m; bidirected=true, kws...) + rand_graph(n, m; bidirected=true, seed=-1, kws...) Generate a random (Erdós-Renyi) `GNNGraph` with `n` nodes and `m` edges. @@ -43,10 +43,10 @@ julia> edge_index(g) ``` """ -function rand_graph(n::Integer, m::Integer; bidirected=true, kws...) +function rand_graph(n::Integer, m::Integer; bidirected=true, seed=-1, kws...) if bidirected @assert iseven(m) "Need even number of edges for bidirected graphs, given m=$m." end m2 = bidirected ? m÷2 : m - return GNNGraph(Graphs.erdos_renyi(n, m2, is_directed=!bidirected); kws...) + return GNNGraph(Graphs.erdos_renyi(n, m2; is_directed=!bidirected, seed); kws...) end From 8f5b8d902f44adfd7aec4d8a185afb897ab70cd3 Mon Sep 17 00:00:00 2001 From: CarloLucibello Date: Mon, 1 Nov 2021 01:42:08 +0100 Subject: [PATCH 3/4] rrul for propagate --- src/msgpass.jl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/msgpass.jl b/src/msgpass.jl index ece15a1d6..93411261f 100644 --- a/src/msgpass.jl +++ b/src/msgpass.jl @@ -152,6 +152,27 @@ function propagate(::typeof(copyxj), g::GNNGraph, ::typeof(+), xi, xj::AbstractM return xj * A end +# Have to define custom rule since CUDA.jl has troubles with some sparse-dense multiplications +function ChainRulesCore.rrule(::typeof(propagate), ::typeof(copyxj), g::GNNGraph, + ::typeof(+), xi, xj::AbstractMatrix, e) + A = adjacency_matrix(g) + y = xj * A + function propagate_pullback(ȳ) + Ȳ = unthunk(ȳ) + dxj = Ȳ * A' + return NoTangent(), NoTangent(), NoTangent(), NoTangent(), NoTangent(), dxj, NoTangent() + end + + function propagate_pullback(ȳ::CuMatrix) + Ȳ = unthunk(ȳ) + dxj = CuArray((A * Ȳ')') + return NoTangent(), NoTangent(), NoTangent(), NoTangent(), NoTangent(), dxj, NoTangent() + end + + y, propagate_pullback +end + + # ## avoid the fast path on gpu until we have better cuda support # function propagate(::typeof(copyxj), g::GNNGraph{<:Union{COO_T,SPARSE_T}}, ::typeof(+), xi, xj::AnyCuMatrix, e) # propagate((xi,xj,e) -> copyxj(xi,xj,e), g, +, xi, xj, e) From 277f3ce1c09bd31d556561a02a276b67cbabaeb6 Mon Sep 17 00:00:00 2001 From: CarloLucibello Date: Mon, 1 Nov 2021 01:56:24 +0100 Subject: [PATCH 4/4] pr perf --- perf/pr_2021_11_01_arrakis.jld2 | Bin 0 -> 14178 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 perf/pr_2021_11_01_arrakis.jld2 diff --git a/perf/pr_2021_11_01_arrakis.jld2 b/perf/pr_2021_11_01_arrakis.jld2 new file mode 100644 index 0000000000000000000000000000000000000000..c55c1997477020821eb751de7c713cb4f6ea4a64 GIT binary patch literal 14178 zcmeGj32;2ZxC5n7PbH_bX^Kp6gd_U*U~~P@IV(8SfJKL*zW(j;pZ_k&QHNs?G{~?>EHcc z_v`N0@BR1s!&rC0kU?dhI&Xzze5kt0<8XU|9!G&cP~!;>bkutTbya?!BRwr6Ekig` zWtWUJXWC$gGiy*;RnSp5x}UfT|FaQzc&xi1IiXX6#cmhEBHcJoP&QQ3yR#&|^T9a} zP8#c;Vi6BnM+(ge-5t>4^VE2S$Zj1cQh-W_zc~04!Y*n(0cZt0TZI?}OCZf%RURbTO6`e6>+&}?)Rg(Fb&ieN6O8TSd_gD0 zsu{JDIpWUNvg2XG@!{n20s&8h?AM&y8`*b*rz$|>*?&fV3#C1?jTHQeaDvi~p(()G zs!JQ^tMJ~DNbT)tJVX11M{G)>evRTh)d!nKUNuH4nc9;`j~+*ygA$MKN=s_z*)67i zt%y(IlJkdh{>oAQV9;Mf{ZgnOrWAs%KhM|Dn)u`{@yqIh0Z)0*5a%}3kLT&d4pPsE zjuN+}_U4ilWAcjg{l0q9E&{)O1b&AI{A*Nxevz)fV+8)S5%`@V@H-RVO7@rg{rXdv z2>h-Q_}4|?cZtqV`7lRZ#8s&`Ru0 z?VJZJ@2;xx`taRWCj--m`f(oX`@s}X5PEnpkNZ+P=NYf#RN`}c{b9 ze-NXlbZU>6YQ>G-a=))ahz#QMJl)(~c346rzVjSXK8yHlkG{T({J~jO zzM0r#2({C46{~tkxgQAsuBW7(O26e@B>ybMXMjV}L#ZG8!Txi;4Wss-i(|Gj-kDxf z|H$4F*AO1A^kcgqv2}Qra1Qm0^ghU?cD7r6-;AJk-e-%XdNGptt%#l(@Klt}tPPb0 z{iWr#A>pF-X56m7rG?NGezm8;iyL~LGEOt_N5UHK)E2Q>I<0Ji> z7_nU{D5CaA^N2A(IK&HAl2?DPM&N1$u14T$1g=Kl|0)8qPO-#l7n-Sx1D>zA;(!$h z{9_u=0!jH;wn)~?7IOnt!JyX{u5f|JTo%JOu2BsR(JCjtP?~tFwVlwy6;EO+kYIU& z$Fah*3isbVd7&J~-3)B7wurMyW2_0nda>4Ghf}6aIKZom>Y(E#aBTlOKT36PTZhUD z5FWfMs{(a!I|%1RoxC5Uq}yfL2TzVMd1jmlynGuY2)aa4F7i^KmXrkJd#fu-E1-md zr?I*VH6{yGflv;C55$rZVz=-6pQI~kN@Sp-n}!zah5lb!vf;5utx83R0Y zSNr|9hiWg@x3DT!C+wz@5B2bD@Q?lx&#t!E9;zKI{04(^u}V4%OgA zn2iNS6&8Vn9cz4-%R4M3v3FcnBg1rL);pGUwv|td>eo1>ce%=p>bZWLy4m68XA>Po zJzW`3rAhHFnG*MzHffqvQ*G^eaenKuUc~__^H(&OoLfBDv-f;SkORu!>+jl|AcxHg zXMKnEvYQ>EXoD%?I@NdANd9f3YHyL^JFLUOYS;`M&=R>eTU{nn*quc7jXosB_H{WR+hg}=R`E(kZ_ij7TY@o94^`3KbcF^Jv8%tFk7R*>3`fl8RvF(H> zuU=Z$e*it7FYrChIfJ{ck;dKM zNaG9|X-s}2jZ4-@<8E!FadwR~&aUOxe;Fwq_y)eY36I;oSmzCa!^cER1J3Tsk2j@v{;|)_HI~ru`JRC=NQ> zXAGJX=)~Ll(EbwBc4d4c^#XAfK080{Ch3Fut>3s6V(5&onL$9uu=Q_Tb^MGRzRueJ z;-SJ9s)3R^1QX+lA*}HFgx$!9DVD~w1;dV*qNW3q4MaJ=aEI^JZX7ML zcdYJ@r!9S8gl?&r^dyNr>-6M?l>w(-*25DmjmwaaG8c%h%x1T!Dlzea`|B9XM4R)~BD0GB;|f#)B{qkH@fIq@LZ~q`CRXS|T$W zgNd09j{P^|YRvUaFu|DXgMl`i>--V$rurSP$$*5t*}dBxbOZ3s?nnCfE1c_$=X7|g zd6VL5{*__OFxc-dff!yH-?#^tMX|OUzBB=Ng>KCox>$A}NQVg60Wq=VQaq6|cC!c)ly&qiH?0e~3X;j!OP{sBYQJLz{DO2^Af}j;7|^mf)g}wvs{G7v8RMQASh6Y$YuRt-p1m_I1~f zHklB6sKdv*&Mcp&9m}f=L=jU%a@&DzmeTHRv}4Z&7D%xxb0~ik4`u-WBs>7mN&sbq zmlEDXxH(-@M-twDe%`VuZmB`k*Slk`miE4Fp9!%_gfnxs%?}*8%Y<0N(LNdWZg*)@ zw*2a=C}L{t=gvH~G4XMWmbdK9aZ$v=`u3-Qw(q5_X40|E{cpejLTZDyttQJ%g4|qK zJ#uaGNmtRjZabqGsrq){sm<^1eX_sSF6Y&@QtZm)ReL;R0~8XzpYR)mj}uO%az`oQ z-w+-e`aU~~TWah-pVw~fnB4ocFApp%i6W*(qjW||>fVXXv{#$ORhkg{WaZ%ut2bBM zcIsSz6Jl+@Timd^e2!)*OxzMhO!aNWZ?`$t9jMnrX$eIp#9po)k?_&+$FvuZ+^`{v zm>QB!5AU%%*LKuS%)e!{39;qNlb%?3aH96euFzvr?8@X-4PI6NUMBnr;Wl^!1{g

Kr`f8a27OFcFPp|A9kyw?FNAx7S~(dJy)_q{e>J@xJ{# zcR1tfG2z%A6QgC{+cu%~9qqIcFKm6lgxJze-wZEabllbEz|B;yipj{2*R5EeT6oH} zY2!-G#Av%`tzNkOtwgPK)shSoVqfg(@yWl+a z-+9{+k43YtNmzPn@gZ09sjb#NYJ1JKXvOO6SsyNRRc5X_UzKFn95Xf~RGvHS+WF`F zY0sbi+*SQ-i_JGy9du2~dF6F!cT88$7LbeE^^6oE2e9#Omkac){gv~jx~7+(S5S$# zQL(KM-5o2IY~U$gLz z!)}j&XTI~SX|d!!>26GF@XH3P841rAO@u$j7cViP2EUZE8W%icF$Q5*;k37#s73>E zyhk9k>@HzrV#6C49)FgojtI3CFj1@vr zueDU9i)p5e*1$^BYvHKom69KvO|)h-h+{1RVdazlbyCfi#><{FLS8E`lro;iNq(wP z8Ww;WW9Y{KZA^@@3Ox>JV{pbR^aP-dUc-WGL{9|Tn7>6p7x~UvbvNP;#aXb9EBHUR zFiz03As(uRaVc^3bO{-}NELo2e^FQXgEC!XLultQEeKKT<3Fq5Sz*;Sb|4 zpb9^mza=*2XQcBHN8xAim0RIwaz#tw59TVQ!q4KeqrxA;l^lidk literal 0 HcmV?d00001