Add Win32::HttpGetFile

craigberry · craigberry · commit 9d421144abee · 2021-12-20T12:30:53.000-06:00
Discussions around mitigating the recent CVEs against PAUSE/CPAN (CVE-2020-16154, CVE-2020-16155, and CVE-2020-16156) included the desire to make Perl core able to do secure downloads out of the box, and further discussion that mechanisms based on OpenSSL, curl, or wget may not be available on Windows. See, for example: https://www.nntp.perl.org/group/perl.perl5.porters/2021/12/msg262180.html This commit adds a native download function to the Win32 module based on the WinHttp library, a function that could in turn be used by CPAN.pm, etc. N.B. The autoproxy detection code has not been tested since I don't have a set-up where I can do that.
diff --git a/Makefile.PL b/Makefile.PL
@@ -22,10 +22,10 @@ my %param = (
 $param{NO_META} = 1 if eval "$ExtUtils::MakeMaker::VERSION" >= 6.10_03;
 
 if ($^O eq 'cygwin') {
-    $param{LIBS} = ['-L/lib/w32api -lole32 -lversion -luserenv -lnetapi32']
+    $param{LIBS} = ['-L/lib/w32api -lole32 -lversion -luserenv -lnetapi32 -lwinhttp']
 }
 else {
-    $param{LIBS} = ['-luserenv']
+    $param{LIBS} = ['-luserenv -lwinhttp']
 }
 
 my $test_requires = $ExtUtils::MakeMaker::VERSION >= 6.64
diff --git a/Win32.pm b/Win32.pm
@@ -1472,6 +1472,12 @@ instead.
 Loads the DLL LIBRARYNAME and calls the function
 DllUnregisterServer.
 
+=item Win32::HttpGetFile(URL, FILENAME)
+
+Uses the WinHttp library to download the file specified by the URL
+parameter to the local file specified by FILENAME. Only http and https
+protocols are supported.  Authentication is not supported.
+
 =back
 
 =head1 CAVEATS
diff --git a/Win32.xs b/Win32.xs
@@ -7,6 +7,7 @@
 #include <wchar.h>
 #include <userenv.h>
 #include <lm.h>
+#include <winhttp.h>
 
 #define PERL_NO_GET_CONTEXT
 #include "EXTERN.h"
@@ -1682,6 +1683,248 @@ XS(w32_IsDeveloperModeEnabled)
     XSRETURN_NO;
 }
 
+
+XS(w32_HttpGetFile)
+{
+    dXSARGS;
+    WCHAR *url = NULL, *file = NULL, *hostName = NULL, *urlPath = NULL;
+    DWORD dwSize = 0;
+    DWORD dwDownloaded = 0;
+    DWORD dwBytesWritten = 0;
+    LPSTR pszOutBuffer;
+    BOOL  bResults = FALSE;
+    HINTERNET  hSession = NULL,
+               hConnect = NULL,
+               hRequest = NULL;
+    HANDLE hOut = NULL;
+    BOOL   bParsed = FALSE,
+           bAborted = FALSE,
+           bFileError = FALSE;
+    DWORD error = 0;
+    URL_COMPONENTS urlComp;
+    LPCWSTR acceptTypes[] = { L"*/*", NULL };
+    WINHTTP_AUTOPROXY_OPTIONS  AutoProxyOptions;
+    WINHTTP_PROXY_INFO         ProxyInfo;
+    DWORD                      cbProxyInfoSize = sizeof(ProxyInfo);
+
+    if (items != 2)
+        croak("usage: Win32::HttpGetFile($url, $file)");
+
+    url = sv_to_wstr(aTHX_ ST(0));
+    file = sv_to_wstr(aTHX_ ST(1));
+
+    /* Initialize the URL_COMPONENTS structure, setting the required
+     * component lengths to non-zero so that they get populated.
+     */
+    ZeroMemory(&urlComp, sizeof(urlComp));
+    urlComp.dwStructSize = sizeof(urlComp);
+    urlComp.dwSchemeLength    = (DWORD)-1;
+    urlComp.dwHostNameLength  = (DWORD)-1;
+    urlComp.dwUrlPathLength   = (DWORD)-1;
+    urlComp.dwExtraInfoLength = (DWORD)-1;
+
+    /* Parse the URL. */
+    bParsed = WinHttpCrackUrl(url, (DWORD)wcslen(url), 0, &urlComp);
+
+    /* Only support http and htts, not ftp, gopher, etc. */
+    if (bParsed
+        && !(urlComp.nScheme == INTERNET_SCHEME_HTTPS
+             || urlComp.nScheme == INTERNET_SCHEME_HTTP)) {
+        SetLastError(12006); /* not a recognized protocol */
+        bParsed = FALSE;
+    }
+
+    if (bParsed) {
+        New(0, hostName,  urlComp.dwHostNameLength + 1, WCHAR);
+        wcsncpy(hostName, urlComp.lpszHostName, urlComp.dwHostNameLength);
+        hostName[urlComp.dwHostNameLength] = 0;
+
+        New(0, urlPath,  urlComp.dwUrlPathLength + urlComp.dwExtraInfoLength + 1, WCHAR);
+        wcsncpy(urlPath, urlComp.lpszUrlPath, urlComp.dwUrlPathLength + urlComp.dwExtraInfoLength);
+        urlPath[urlComp.dwUrlPathLength + urlComp.dwExtraInfoLength] = 0;
+
+        /* Use WinHttpOpen to obtain a session handle. */
+        hSession = WinHttpOpen(L"Perl",
+                               WINHTTP_ACCESS_TYPE_NO_PROXY,
+                               WINHTTP_NO_PROXY_NAME,
+                               WINHTTP_NO_PROXY_BYPASS,
+                               0);
+    }
+
+    /* Specify an HTTP server. */
+    if (hSession)
+        hConnect = WinHttpConnect(hSession,
+                                  hostName,
+                                  urlComp.nPort,
+                                  0);
+
+    /* Create an HTTP request handle. */
+    if (hConnect)
+        hRequest = WinHttpOpenRequest(hConnect,
+                                      L"GET",
+                                      urlPath,
+                                      NULL,
+                                      WINHTTP_NO_REFERER,
+                                      acceptTypes,
+                                      urlComp.nScheme == INTERNET_SCHEME_HTTPS
+                                                      ? WINHTTP_FLAG_SECURE
+                                                      : 0);
+
+    /* Call WinHttpGetProxyForUrl with our target URL. If auto-proxy succeeds,
+     * then set the proxy info on the request handle. If auto-proxy fails,
+     * ignore the error and attempt to send the HTTP request directly to the
+     * target server (using the default WINHTTP_ACCESS_TYPE_NO_PROXY
+     * configuration, which the request handle will inherit from the session).
+     */
+    if (hRequest) {
+        ZeroMemory(&AutoProxyOptions, sizeof(AutoProxyOptions));
+        ZeroMemory(&ProxyInfo, sizeof(ProxyInfo));
+        AutoProxyOptions.dwFlags = WINHTTP_AUTOPROXY_AUTO_DETECT;
+        AutoProxyOptions.dwAutoDetectFlags =
+                                    WINHTTP_AUTO_DETECT_TYPE_DHCP |
+                                    WINHTTP_AUTO_DETECT_TYPE_DNS_A;
+        AutoProxyOptions.fAutoLogonIfChallenged = TRUE;
+
+        if(WinHttpGetProxyForUrl(hSession,
+                                url,
+                                &AutoProxyOptions,
+                                &ProxyInfo)) {
+            if(!WinHttpSetOption(hRequest,
+                                WINHTTP_OPTION_PROXY,
+                                &ProxyInfo,
+                                cbProxyInfoSize)) {
+                bAborted = TRUE;
+                Perl_warn(aTHX_ "Win32::HttpGetFile: setting proxy options failed");
+            }
+            Safefree(ProxyInfo.lpszProxy);
+            Safefree(ProxyInfo.lpszProxyBypass);
+        }
+    }
+
+    /* Send a request. */
+    if (hRequest && !bAborted)
+        bResults = WinHttpSendRequest(hRequest,
+                                      WINHTTP_NO_ADDITIONAL_HEADERS,
+                                      0,
+                                      WINHTTP_NO_REQUEST_DATA,
+                                      0,
+                                      0,
+                                      0);
+
+    /* End the request. */
+    if (bResults)
+        bResults = WinHttpReceiveResponse(hRequest, NULL);
+
+    /* Create output file for download. */
+    if (bResults) {
+        hOut = CreateFileW(file,
+                           GENERIC_WRITE,
+                           FILE_SHARE_READ | FILE_SHARE_WRITE,
+                           NULL,
+                           CREATE_ALWAYS,
+                           FILE_ATTRIBUTE_NORMAL,
+                           NULL);
+
+        if (!hOut || hOut == INVALID_HANDLE_VALUE)
+            bFileError = TRUE;
+    }
+
+    /* Keep checking for data until there is nothing left. */
+    if (!bFileError && bResults) {
+        do {
+            /* Check for available data. */
+            dwSize = 0;
+            if (!WinHttpQueryDataAvailable(hRequest, &dwSize)) {
+                bAborted = TRUE;
+                break;
+            }
+
+            /* No more available data. */
+            if (!dwSize)
+                break;
+
+            /* Allocate space for the buffer. */
+            New(0, pszOutBuffer, dwSize + 1, char);
+            if (!pszOutBuffer) {
+                bAborted = TRUE;
+                break;
+            }
+
+            /* Read the Data. */
+            ZeroMemory(pszOutBuffer, dwSize+1);
+
+            if (!WinHttpReadData(hRequest,
+                                 (LPVOID)pszOutBuffer,
+                                 dwSize,
+                                 &dwDownloaded)) {
+                bAborted = TRUE;
+                Safefree(pszOutBuffer);
+                break;
+            }
+
+            /* Write what we just read to the output file */
+            if (!WriteFile(hOut,
+                           pszOutBuffer,
+                           dwDownloaded,
+                           &dwBytesWritten,
+                           NULL)) {
+                bAborted = TRUE;
+                bFileError = TRUE;
+                Safefree(pszOutBuffer);
+                break;
+            }
+
+            Safefree(pszOutBuffer);
+
+            /* This condition would only be reached if WinHttpQueryDataAvailable
+             * said there are more data to read but WinHttpReadData didn't get any.
+             */
+            if (!dwDownloaded)
+                break;
+        }
+        while (dwSize > 0);
+    }
+    else {
+        bAborted = TRUE;
+    }
+
+    /* Clean-up may lose this. */
+    if (bAborted)
+        error = GetLastError();
+
+    /* Close any open handles. */
+    if (hOut) CloseHandle(hOut);
+    if (hRequest) WinHttpCloseHandle(hRequest);
+    if (hConnect) WinHttpCloseHandle(hConnect);
+    if (hSession) WinHttpCloseHandle(hSession);
+
+    Safefree(url);
+    Safefree(file);
+    Safefree(hostName);
+    Safefree(urlPath);
+
+    if (bAborted) {
+        char msgbuf[ONE_K_BUFSIZE];
+        DWORD msgFlags = bFileError
+                         ? FORMAT_MESSAGE_FROM_SYSTEM
+                         : FORMAT_MESSAGE_FROM_HMODULE;
+
+        if (FormatMessageA(msgFlags,
+                           GetModuleHandleA("winhttp.dll"),
+                           error,
+                           0,
+                           msgbuf,
+                           sizeof(msgbuf) - 1,
+                           NULL)) {
+            Perl_warn(aTHX_ "Error %lu in Win32::HttpGetFile: %s", error, msgbuf);
+        }
+        SetLastError(error);
+        XSRETURN_NO;
+    }
+
+    XSRETURN_YES;
+}
+
 MODULE = Win32            PACKAGE = Win32
 
 PROTOTYPES: DISABLE
@@ -1756,5 +1999,6 @@ BOOT:
 #ifdef __CYGWIN__
     newXS("Win32::SetChildShowWindow", w32_SetChildShowWindow, file);
 #endif
+    newXS("Win32::HttpGetFile", w32_HttpGetFile, file);
     XSRETURN_YES;
 }
diff --git a/t/HttpGetFile.t b/t/HttpGetFile.t
@@ -0,0 +1,31 @@
+use strict;
+use warnings;
+use Test;
+use Win32;
+use Digest::SHA;
+
+my $tmpfile = "http-download-test-$$.tgz";
+END { 1 while unlink $tmpfile; }
+
+# We may not always have an internet connection, so don't
+# attempt remote connections unless the user has done
+#   set PERL_WIN32_INTERNET_OK=1
+plan tests => $ENV{PERL_WIN32_INTERNET_OK} ? 6 : 4;
+
+ok(Win32::HttpGetFile('nonesuch://example.com', 'NUL:'), "", "'nonesuch://' is not a real protocol");
+ok(Win32::GetLastError(), '12006', "correct error code for unrecognized protocol");
+ok(Win32::HttpGetFile('http://!#@!&@$', 'NUL:'), "", "invalid URL");
+ok(Win32::GetLastError(), '12005', "correct error code for invalid URL");
+
+if ($ENV{PERL_WIN32_INTERNET_OK}) {
+    # The digest for version 0.57 should obviously stay the same even after new versions are released
+    ok(Win32::HttpGetFile('https://cpan.metacpan.org/authors/id/J/JD/JDB/Win32-0.57.tar.gz', $tmpfile),
+       '1',
+       "successfully downloaded a tarball");
+
+    my $sha = Digest::SHA->new('sha1');
+    $sha->addfile($tmpfile, 'b');
+    ok($sha->hexdigest,
+       '44a6d7d1607d7267b0dbcacbb745cec204f1c1a4',
+       "downloaded tarball has correct digest");
+}

Original file line number	Diff line number	Diff line change
`@@ -22,10 +22,10 @@ my %param = (`
`22`	`22`	`$param{NO_META} = 1 if eval "$ExtUtils::MakeMaker::VERSION" >= 6.10_03;`
`23`	`23`
`24`	`24`	`if ($^O eq 'cygwin') {`
`25`		`- $param{LIBS} = ['-L/lib/w32api -lole32 -lversion -luserenv -lnetapi32']`
	`25`	`+ $param{LIBS} = ['-L/lib/w32api -lole32 -lversion -luserenv -lnetapi32 -lwinhttp']`
`26`	`26`	`}`
`27`	`27`	`else {`
`28`		`- $param{LIBS} = ['-luserenv']`
	`28`	`+ $param{LIBS} = ['-luserenv -lwinhttp']`
`29`	`29`	`}`
`30`	`30`
`31`	`31`	`my $test_requires = $ExtUtils::MakeMaker::VERSION >= 6.64`