HTML format: offline_version parameter

src/html/HTMLWriter.jl

@@ -484,6 +484,7 @@ struct HTML <: Documenter.Writer
     size_threshold_ignore :: Vector{String}
     example_size_threshold :: Int
     inventory_version ::  Union{String,Nothing}
+    offline_version :: Bool
 
     function HTML(;
             prettyurls    :: Bool = true,
@@ -513,6 +514,7 @@ struct HTML <: Documenter.Writer
             # and leaves a buffer before hitting `size_threshold_warn`.
             example_size_threshold :: Union{Integer, Nothing} = 8 * 2^10, # 8 KiB
             inventory_version = nothing,
+            offline_version = false,
 
             # deprecated keywords
             edit_branch   :: Union{String, Nothing, Default} = Default(nothing),
@@ -568,7 +570,7 @@ struct HTML <: Documenter.Writer
             collapselevel, sidebar_sitename, highlights, mathengine, description, footer,
             ansicolor, lang, warn_outdated, prerender, node, highlightjs,
             size_threshold, size_threshold_warn, size_threshold_ignore, example_size_threshold,
-            (isnothing(inventory_version) ? nothing : string(inventory_version))
+            (isnothing(inventory_version) ? nothing : string(inventory_version)), offline_version
         )
     end
 end
@@ -596,7 +598,7 @@ function prepare_prerendering(prerender, node, highlightjs, highlights)
         end
         @debug "HTMLWriter: downloading highlightjs"
         r = Documenter.JSDependencies.RequireJS([])
-        RD.highlightjs!(r, highlights)
+        RD.highlightjs!(r, false, "", "", highlights)
         libs = sort!(collect(r.libraries); by = first) # puts highlight first
         key = join((x.first for x in libs), ',')
         highlightjs = get!(HLJSFILES, key) do
@@ -749,12 +751,15 @@ function render(doc::Documenter.Document, settings::HTML=HTML())
     if isfile(joinpath(doc.user.source, "assets", "documenter.js"))
         @warn "not creating 'documenter.js', provided by the user."
     else
-        r = JSDependencies.RequireJS([
-            RD.jquery, RD.jqueryui, RD.headroom, RD.headroom_jquery,
-        ])
-        RD.mathengine!(r, settings.mathengine)
+        r = JSDependencies.RequireJS([RD.process_remote(url, settings.offline_version, joinpath(doc.user.build, "assets", "cdn"), joinpath(doc.user.build, "assets")) for url in [
+            RD.jquery, 
+            RD.jqueryui, 
+            RD.headroom, 
+            RD.headroom_jquery,
+        ]])
+        RD.mathengine!(r, settings.mathengine, settings.offline_version, joinpath(doc.user.build, "assets", "cdn"), joinpath(doc.user.build, "assets"))
         if !settings.prerender
-            RD.highlightjs!(r, settings.highlights)
+            RD.highlightjs!(r, settings.offline_version, joinpath(doc.user.build, "assets", "cdn"), joinpath(doc.user.build, "assets"), settings.highlights)
         end
         for filename in readdir(joinpath(ASSETS, "js"))
             path = joinpath(ASSETS, "js", filename)
@@ -952,12 +957,12 @@ function render_head(ctx, navnode)
         default_site_description(ctx)
     end
 
-    css_links = [
+    css_links = [RD.process_remote(url, ctx.settings.offline_version, joinpath(ctx.doc.user.build, "assets", "cdn"), ctx.doc.user.build) for url in [
         RD.lato,
         RD.juliamono,
         RD.fontawesome_css...,
         RD.katex_css,
-    ]
+    ]]
 
     head(
         meta[:charset=>"UTF-8"],
@@ -991,7 +996,7 @@ function render_head(ctx, navnode)
 
         script("documenterBaseURL=\"$(relhref(src, "."))\""),
         script[
-            :src => RD.requirejs_cdn,
+            :src => RD.process_remote(RD.requirejs_cdn, ctx.settings.offline_version, joinpath(ctx.doc.user.build, "assets", "cdn"), ctx.doc.user.build),
             Symbol("data-main") => relhref(src, ctx.documenter_js)
         ],
         script[:src => relhref(src, ctx.search_index_js)],

src/html/RD.jl

@@ -1,6 +1,8 @@
 "Provides a namespace for remote dependencies."
 module RD
     using JSON: JSON
+    using Base64
+    using ....Documenter: hascurl
     using ....Documenter.JSDependencies: RemoteLibrary, Snippet, RequireJS, jsescape, json_jsescape
     using ..HTMLWriter: KaTeX, MathJax, MathJax2, MathJax3
 
@@ -29,23 +31,23 @@ module RD
 
     # highlight.js
     "Add the highlight.js dependencies and snippet to a [`RequireJS`](@ref) declaration."
-    function highlightjs!(r::RequireJS, languages = String[])
+    function highlightjs!(r::RequireJS, offline_version::Bool, build_path::AbstractString, origin_path=build_path, languages = String[])
         # NOTE: the CSS themes for hightlightjs are compiled into the Documenter CSS
         # When updating this dependency, it is also necessary to update the the CSS
         # files the CSS files in assets/html/scss/highlightjs
         hljs_version = "11.8.0"
-        push!(r, RemoteLibrary(
+        push!(r, process_remote(RemoteLibrary(
             "highlight",
             "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/$(hljs_version)/highlight.min.js"
-        ))
+        ), offline_version, build_path, origin_path))
         languages = ["julia", "julia-repl", languages...]
         for language in languages
             language = jsescape(language)
-            push!(r, RemoteLibrary(
+            push!(r, process_remote(RemoteLibrary(
                 "highlight-$(language)",
                 "https://cdnjs.cloudflare.com/ajax/libs/highlight.js/$(hljs_version)/languages/$(language).min.js",
                 deps = ["highlight"]
-            ))
+            ), offline_version, build_path, origin_path))
         end
         push!(r, Snippet(
             vcat(["jquery", "highlight"], ["highlight-$(jsescape(language))" for language in languages]),
@@ -61,16 +63,16 @@ module RD
     # MathJax & KaTeX
     const katex_version = "0.16.8"
     const katex_css = "https://cdnjs.cloudflare.com/ajax/libs/KaTeX/$(katex_version)/katex.min.css"
-    function mathengine!(r::RequireJS, engine::KaTeX)
-        push!(r, RemoteLibrary(
+    function mathengine!(r::RequireJS, engine::KaTeX, offline_version::Bool, build_path, origin_path=build_path)
+        push!(r, process_remote(RemoteLibrary(
             "katex",
             "https://cdnjs.cloudflare.com/ajax/libs/KaTeX/$(katex_version)/katex.min.js"
-        ))
-        push!(r, RemoteLibrary(
+        ), offline_version, build_path, origin_path))
+        push!(r, process_remote(RemoteLibrary(
             "katex-auto-render",
             "https://cdnjs.cloudflare.com/ajax/libs/KaTeX/$(katex_version)/contrib/auto-render.min.js",
             deps = ["katex"],
-        ))
+        ), offline_version, build_path, origin_path))
         push!(r, Snippet(
             ["jquery", "katex", "katex-auto-render"],
             ["\$", "katex", "renderMathInElement"],
@@ -84,8 +86,9 @@ module RD
             """
         ))
     end
-    function mathengine!(r::RequireJS, engine::MathJax2)
+    function mathengine!(r::RequireJS, engine::MathJax2, offline_version::Bool, build_path, origin_path=build_path)
         url = isempty(engine.url) ? "https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.9/MathJax.js?config=TeX-AMS_HTML" : engine.url
+        url = process_remote(url, offline_version, build_path, origin_path)
         push!(r, RemoteLibrary(
             "mathjax",
             url,
@@ -97,8 +100,9 @@ module RD
             """
         ))
     end
-    function mathengine!(r::RequireJS, engine::MathJax3)
+    function mathengine!(r::RequireJS, engine::MathJax3, offline_version::Bool, build_path, origin_path=build_path)
         url = isempty(engine.url) ? "https://cdnjs.cloudflare.com/ajax/libs/mathjax/3.2.2/es5/tex-svg.js" : engine.url
+        url = process_remote(url, offline_version, build_path, origin_path)
         push!(r, Snippet([], [],
             """
             window.MathJax = $(json_jsescape(engine.config, 2));
@@ -113,4 +117,59 @@ module RD
         ))
     end
     mathengine(::RequireJS, ::Nothing) = nothing
+
+    process_remote(dep, offline_version::Bool, build_path, origin_path=build_path) = offline_version ? _process(dep, build_path, origin_path) : dep
+    _process(dep::RemoteLibrary, build_path, origin_path) = RemoteLibrary(dep.name, _process(dep.url, build_path, origin_path); deps = dep.deps, exports = dep.exports)
+
+    function _download_file_content(url::AbstractString)
+        cmd = `curl $url -s --output -`
+        try
+            return read(cmd, String)
+        catch err
+            @error "$cmd failed: ", err
+            error()
+            return
+        end
+    end
+
+    function _process(url::AbstractString, output_path, origin_path)
+        if !hascurl()
+            @error "offline_version requires curl."
+            error()
+        end
+        result = _download_file_content(url)
+        filename = split(url, "/")[end]
+        filepath = joinpath(output_path, filename)
+        if !isfile(filepath)
+            mkpath(dirname(filepath))
+            open(filepath, "w") do f
+                if splitext(filepath)[end] == ".css"
+                    result = _process_downloaded_css(result, url)
+                end
+                write(f, result)
+            end
+        end
+
+        return relpath(filepath, origin_path*"/")
+    end
+
+    const font_ext_to_type = Dict(
+        ".ttf" => "truetype",
+        ".eot" => "embedded-opentype",
+        ".eot?#iefix" => "embedded-opentype",
+        ".svg#webfont" => "svg",
+        ".woff" => "woff",
+        ".woff2" => "woff2",
+    )
+
+    function _process_downloaded_css(file_content, origin_url)
+        url_regex = r"url\(([^)]+)\)"
+        replace(file_content, url_regex => s -> begin
+            rel_url = match(url_regex, s).captures[1]
+            url = normpath(dirname(origin_url), rel_url)
+            font_type = font_ext_to_type[splitext(rel_url)[end]]
+            encoded_file = Base64.base64encode(_download_file_content(url))
+            return "url(data:font/$(font_type);charset=utf-8;base64,$(encoded_file))"
+        end)
+    end
 end

test/examples/make.jl

@@ -18,7 +18,7 @@ EXAMPLE_BUILDS = if haskey(ENV, "DOCUMENTER_TEST_EXAMPLES")
     split(ENV["DOCUMENTER_TEST_EXAMPLES"])
 else
     ["html", "html-meta-custom", "html-mathjax2-custom", "html-mathjax3", "html-mathjax3-custom",
-    "html-local", "html-draft", "html-repo-git", "html-repo-nothing", "html-repo-error",
+    "html-local", "html-offline", "html-draft", "html-repo-git", "html-repo-nothing", "html-repo-error",
     "html-sizethreshold-defaults-fail", "html-sizethreshold-success", "html-sizethreshold-ignore-success", "html-sizethreshold-override-fail", "html-sizethreshold-ignore-success", "html-sizethreshold-ignore-fail",
     "latex_texonly", "latex_simple_texonly", "latex_showcase_texonly", "html-pagesonly"]
 end
@@ -476,6 +476,36 @@ else
     nothing
 end
 
+# HTML: offline_version
+examples_html_offline_doc = if "html-offline" in EXAMPLE_BUILDS
+    @info("Building mock package docs: HTMLWriter / offline build")
+    @quietly makedocs(
+        debug = true,
+        root  = examples_root,
+        build = "builds/html-offline",
+        doctestfilters = [r"Ptr{0x[0-9]+}"],
+        sitename = "Documenter example",
+        pages = htmlbuild_pages,
+        expandfirst = expandfirst,
+        repo = "https://dev.azure.com/org/project/_git/repo?path={path}&version={commit}{line}&lineStartColumn=1&lineEndColumn=1",
+        linkcheck = true,
+        linkcheck_ignore = [r"(x|y).md", "z.md", r":func:.*"],
+        format = Documenter.HTML(
+            assets = [
+                "assets/custom.css"
+            ],
+            offline_version = true,
+            footer = nothing,
+        ),
+        # TODO: example_block failure only happens on windows, so that's not actually expected
+        warnonly = [:doctest, :footnote, :cross_references, :linkcheck, :example_block, :eval_block],
+    )
+else
+    @info "Skipping build: HTML/offline"
+    @debug "Controlling variables:" EXAMPLE_BUILDS get(ENV, "DOCUMENTER_TEST_EXAMPLES", nothing)
+    nothing
+end
+
 # HTML: A few simple builds testing the repo keyword fallbacks
 macro examplebuild(name, block)
     docvar = Symbol("examples_html_", replace(name, "-" => "_"), "_doc")

test/examples/tests.jl

@@ -423,6 +423,25 @@ end
         end
     end
 
+
+    @testset "HTML: offline" begin
+        doc = Main.examples_html_offline_doc
+
+        @test isa(doc, Documenter.Documenter.Document)
+
+        let build_dir = joinpath(examples_root, "builds", "html-offline")
+
+            index_html = read(joinpath(build_dir, "index.html"), String)
+            @test occursin("<link href=\"assets/cdn/lato-font.min.css\" rel=\"stylesheet\" type=\"text/css\"/>", index_html)
+
+            # Assets
+            @test joinpath(build_dir, "assets", "documenter.js") |> isfile
+            @test joinpath(build_dir, "assets", "cdn", "lato-font.min.css") |> isfile
+            documenterjs = String(read(joinpath(build_dir, "assets", "documenter.js")))
+            @test occursin("'jquery': 'cdn/jquery.min'", documenterjs)
+        end
+    end
+
     @testset "HTML: pagesonly" begin
         doc = Main.examples_html_pagesonly_doc