Merge pull request #37340 from JuliaLang/sk/downloader

add Downloads stdlib

Author: StefanKarpinski

Make.inc

@@ -49,6 +49,7 @@ USE_SYSTEM_LIBUV:=0
 USE_SYSTEM_UTF8PROC:=0
 USE_SYSTEM_MBEDTLS:=0
 USE_SYSTEM_LIBSSH2:=0
+USE_SYSTEM_NGHTTP2:=0
 USE_SYSTEM_CURL:=0
 USE_SYSTEM_LIBGIT2:=0
 USE_SYSTEM_PATCHELF:=0
@@ -1099,7 +1100,7 @@ USE_BINARYBUILDER ?= 0
 endif
 
 # This is the set of projects that BinaryBuilder dependencies are hooked up for.
-BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP
+BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP
 define SET_BB_DEFAULT
 # First, check to see if BB is disabled on a global setting
 ifeq ($$(USE_BINARYBUILDER),0)

Makefile

@@ -170,6 +170,7 @@ JL_PRIVATE_LIBS-$(USE_SYSTEM_DSFMT) += libdSFMT
 JL_PRIVATE_LIBS-$(USE_SYSTEM_GMP) += libgmp
 JL_PRIVATE_LIBS-$(USE_SYSTEM_MPFR) += libmpfr
 JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBSSH2) += libssh2
+JL_PRIVATE_LIBS-$(USE_SYSTEM_NGHTTP2) += libnghttp2
 JL_PRIVATE_LIBS-$(USE_SYSTEM_MBEDTLS) += libmbedtls libmbedcrypto libmbedx509
 JL_PRIVATE_LIBS-$(USE_SYSTEM_CURL) += libcurl
 JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBGIT2) += libgit2

NEWS.md

@@ -1,51 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-# file downloading
-
-if Sys.iswindows()
-    function download_powershell(url::AbstractString, filename::AbstractString)
-        ps = joinpath(get(ENV, "SYSTEMROOT", "C:\\Windows"), "System32\\WindowsPowerShell\\v1.0\\powershell.exe")
-        tls12 = "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12"
-        client = "New-Object System.Net.Webclient"
-        # in the following we escape ' with '' (see https://ss64.com/ps/syntax-esc.html)
-        downloadfile = "($client).DownloadFile('$(replace(url, "'" => "''"))', '$(replace(filename, "'" => "''"))')"
-        # PowerShell v3 or later is required for Tls12
-        proc = run(pipeline(`$ps -Version 3 -NoProfile -Command "$tls12; $downloadfile"`; stderr=stderr); wait=false)
-        if !success(proc)
-            if proc.exitcode % Int32 == -393216
-                # appears to be "wrong version" exit code, based on
-                # https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-startup-tasks-common
-                @error "Downloading files requires Windows Management Framework 3.0 or later."
-            end
-            pipeline_error(proc)
-        end
-        return filename
-    end
-end
-
-function find_curl()
-    if Sys.isapple() && Sys.isexecutable("/usr/bin/curl")
-        "/usr/bin/curl"
-    elseif Sys.iswindows() && Sys.isexecutable(joinpath(get(ENV, "SYSTEMROOT", "C:\\Windows"), "System32\\curl.exe"))
-        joinpath(get(ENV, "SYSTEMROOT", "C:\\Windows"), "System32\\curl.exe")
-    elseif !Sys.iswindows() && Sys.which("curl") !== nothing
-        "curl"
-    else
-        nothing
-    end
-end
-
-function download_curl(curl_exe::AbstractString, url::AbstractString, filename::AbstractString)
-    err = PipeBuffer()
-    process = run(pipeline(`$curl_exe -s -S -g -L -f -o $filename $url`, stderr=err), wait=false)
-    if !success(process)
-        error_msg = readline(err)
-        @error "Download failed: $error_msg"
-        pipeline_error(process)
-    end
-    return filename
-end
-
 const DOWNLOAD_HOOKS = Callable[]
 
 function download_url(url::AbstractString)
@@ -55,50 +9,25 @@ function download_url(url::AbstractString)
     return url
 end
 
-function download(url::AbstractString, filename::AbstractString)
-    url = download_url(url)
-    curl_exe = find_curl()
-    if curl_exe !== nothing
-        return download_curl(curl_exe, url, filename)
-    elseif Sys.iswindows()
-        return download_powershell(url, filename)
-    elseif Sys.which("wget") !== nothing
-        try
-            run(`wget -O $filename $url`)
-        catch
-            rm(filename, force=true)  # wget always creates a file
-            rethrow()
-        end
-    elseif Sys.which("busybox") !== nothing
-        try
-            run(`busybox wget -O $filename $url`)
-        catch
-            rm(filename, force=true)  # wget always creates a file
-            rethrow()
-        end
-    elseif Sys.which("fetch") !== nothing
-        run(`fetch -f $filename $url`)
-    else
-        error("No download agent available; install curl, wget, busybox or fetch.")
-    end
-    return filename
-end
-
-function download(url::AbstractString)
-    filename = tempname()
-    download(url, filename)
-end
+Downloads() = require(PkgId(
+        UUID((0xf43a241f_c20a_4ad4, 0x852c_f6b1247861c6)),
+        "Downloads",
+    ))
 
 """
-    download(url::AbstractString, [localfile::AbstractString])
+    download(url::AbstractString, [path::AbstractString = tempname()]) -> path
 
-Download a file from the given url, optionally renaming it to the given local file name. If
-no filename is given this will download into a randomly-named file in your temp directory.
-Note that this function relies on the availability of external tools such as `curl`, `wget`
-or `fetch` to download the file and is provided for convenience. For production use or
-situations in which more options are needed, please use a package that provides the desired
-functionality instead.
+Download a file from the given url, saving it to the location `path`, or if not
+specified, a temporary path. Returns the path of the downloaded file.
 
-Returns the filename of the downloaded file.
+!!! note
+    Since Julia 1.6, this function is deprecated and is just a thin wrapper
+    around `Downloads.download`. In new code, you should use that function
+    directly instead of calling this.
 """
-download(url, filename)
+function download(url::AbstractString, path::AbstractString)
+    invokelatest(Downloads().download, download_url(url), path)
+end
+function download(url::AbstractString)
+    invokelatest(Downloads().download, download_url(url))
+end

base/download.jl

@@ -52,6 +52,10 @@ let
             :Test,
             :REPL,
             :Statistics,
+            :MozillaCACerts_jll,
+            :LibCURL_jll,
+            :LibCURL,
+            :Downloads,
         ]
 
     maxlen = reduce(max, textwidth.(string.(stdlibs)); init=0)

base/sysimg.jl

@@ -12,7 +12,7 @@
 TRIPLETS="i686-linux-gnu x86_64-linux-gnu aarch64-linux-gnu armv7l-linux-gnueabihf powerpc64le-linux-gnu i686-linux-musl x86_64-linux-musl aarch64-linux-musl armv7l-linux-musleabihf x86_64-apple-darwin14 x86_64-unknown-freebsd11.1 i686-w64-mingw32 x86_64-w64-mingw32"
 
 # These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded:
-BB_PROJECTS="mbedtls libssh2 mpfr curl libgit2 pcre libuv unwind osxunwind dsfmt objconv p7zip zlib suitesparse openlibm"
+BB_PROJECTS="mbedtls libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind osxunwind dsfmt objconv p7zip zlib suitesparse openlibm"
 BB_GCC_EXPANDED_PROJECTS="openblas"
 BB_CXX_EXPANDED_PROJECTS="gmp llvm"
 

contrib/refresh_bb_tarballs.sh

@@ -111,11 +111,13 @@ ifeq ($(USE_SYSTEM_LIBSSH2), 0)
 DEP_LIBS += libssh2
 endif
 
-ifneq ($(OS), WINNT)
+ifeq ($(USE_SYSTEM_NGHTTP2), 0)
+DEP_LIBS += nghttp2
+endif
+
 ifeq ($(USE_SYSTEM_CURL), 0)
 DEP_LIBS += curl
 endif
-endif
 
 DEP_LIBS += libgit2
 endif # USE_SYSTEM_LIBGIT2
@@ -178,7 +180,7 @@ install: $(addprefix install-, $(DEP_LIBS))
 cleanall: $(addprefix clean-, $(DEP_LIBS))
 distcleanall: $(addprefix distclean-, $(DEP_LIBS))
 	rm -rf $(build_prefix)
-getall: get-llvm get-libuv get-pcre get-openlibm get-dsfmt get-openblas get-lapack get-suitesparse get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-objconv get-mbedtls get-libssh2 get-curl get-libgit2 get-libwhich
+getall: get-llvm get-libuv get-pcre get-openlibm get-dsfmt get-openblas get-lapack get-suitesparse get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-objconv get-mbedtls get-libssh2 get-nghttp2 get-curl get-libgit2 get-libwhich
 
 include $(SRCDIR)/llvm.mk
 include $(SRCDIR)/libuv.mk
@@ -195,6 +197,7 @@ include $(SRCDIR)/mpfr.mk
 include $(SRCDIR)/patchelf.mk
 include $(SRCDIR)/mbedtls.mk
 include $(SRCDIR)/libssh2.mk
+include $(SRCDIR)/nghttp2.mk
 include $(SRCDIR)/curl.mk
 include $(SRCDIR)/libgit2.mk
 include $(SRCDIR)/libwhich.mk

deps/Makefile

@@ -24,8 +24,10 @@ MBEDTLS_VER = 2.16.8
 MBEDTLS_BB_REL = 0
 LIBSSH2_VER = 1.9.0
 LIBSSH2_BB_REL = 1
-CURL_VER = 7.66.0
-CURL_BB_REL = 1
+CURL_VER = 7.71.1
+CURL_BB_REL = 0
+NGHTTP2_VER = 1.40.0
+NGHTTP2_BB_REL = 2
 LIBGIT2_VER = 1.0.1
 LIBGIT2_BB_REL = 0
 LIBUV_VER = 1.29.1

deps/Versions.make

@@ -0,0 +1 @@
+24a8b8fc2398d20c24a13ce73482a3d7

deps/checksums/Downloads-1a1d2e0a10209512f5b29e585bfd78e7a47f8f61.tar.gz/md5

@@ -0,0 +1 @@
+4b652be535dce6a5cf36e546a31d3221f4c2534d1a3ac710f31f9ed2dfbeabd21b36c0186e4abe12cfee992ff557317aefe2bdeb9cb8b1c44da085b030719329