usr.bin/fetch: Handle custom headers and method for HTTP

Modern APIs often use HTTP method and headers as part of their
interface.

Add functionality to fetch allowing HTTP method and headers to be
specified.  The option names align with those used by curl.

Ability to provide request body data remains unimplemented.

Signed-off-by: Ryan Moeller <[email protected]>

Author: ryan-moeller

usr.bin/fetch/fetch.1

@@ -28,7 +28,7 @@
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .\"
-.Dd October 7, 2023
+.Dd September 27, 2025
 .Dt FETCH 1
 .Os
 .Sh NAME
@@ -43,6 +43,7 @@
 .Op Fl -ca-path= Ns Ar dir
 .Op Fl -cert= Ns Ar file
 .Op Fl -crl= Ns Ar file
+.Op Fl H Ar header
 .Op Fl i Ar file
 .Op Fl -key= Ns Ar file
 .Op Fl N Ar file
@@ -58,6 +59,7 @@
 .Op Fl T Ar seconds
 .Op Fl -user-agent= Ns Ar agent-string
 .Op Fl w Ar seconds
+.Op Fl X Ar method
 .Ar URL ...
 .Nm
 .Op Fl 146AadFlMmnPpqRrsUv
@@ -67,6 +69,7 @@
 .Op Fl -ca-path= Ns Ar dir
 .Op Fl -cert= Ns Ar file
 .Op Fl -crl= Ns Ar file
+.Op Fl H Ar header
 .Op Fl i Ar file
 .Op Fl -key= Ns Ar file
 .Op Fl N Ar file
@@ -82,6 +85,7 @@
 .Op Fl T Ar seconds
 .Op Fl -user-agent= Ns Ar agent-string
 .Op Fl w Ar seconds
+.Op Fl X Ar method
 .Fl h Ar host Fl f Ar file Oo Fl c Ar dir Oc
 .Sh DESCRIPTION
 The
@@ -164,6 +168,11 @@ The file to retrieve is named
 on the remote host.
 This option is deprecated and is provided for backward compatibility
 only.
+.It Fl H Ar header , Fl -header= Ns Ar header
+Send
+.Ar header
+as an additional header line with the HTTP request.
+This option may be specified multiple times.
 .It Fl h Ar host
 The file to retrieve is located on the host
 .Ar host .
@@ -306,6 +315,10 @@ Increase verbosity level.
 When the
 .Fl a
 flag is specified, wait this many seconds between successive retries.
+.It Fl X Ar method , Fl -request= Ns Ar method
+Use the given
+.Ar method
+for HTTP requests.
 .El
 .Pp
 If

usr.bin/fetch/fetch.c

@@ -30,6 +30,7 @@
  */
 
 #include <sys/param.h>
+#include <sys/queue.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
 #include <sys/time.h>
@@ -39,6 +40,7 @@
 #include <errno.h>
 #include <getopt.h>
 #include <signal.h>
+#include <stdbool.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -85,7 +87,10 @@ static int	 v_tty;		/*        stdout is a tty */
 static int	 v_progress;	/*        whether to display progress */
 static pid_t	 pgrp;		/*        our process group */
 static long	 w_secs;	/*    -w: retry delay */
+static char	*method;	/*    -X: HTTP method */
 static int	 family = PF_UNSPEC;	/* -[46]: address family to use */
+static struct http_headers headers = TAILQ_HEAD_INITIALIZER(headers);
+				/*    -H: HTTP headers */
 
 static int	 sigalrm;	/* SIGALRM received */
 static int	 siginfo;	/* SIGINFO received */
@@ -127,6 +132,7 @@ static struct option longopts[] =
 	{ "direct", no_argument, NULL, 'd' },
 	{ "force-restart", no_argument, NULL, 'F' },
 	/* -f not mapped, since it's deprecated */
+	{ "header", required_argument, NULL, 'H' },
 	/* -h not mapped, since it's deprecated */
 	{ "if-modified-since", required_argument, NULL, 'i' },
 	{ "symlink", no_argument, NULL, 'l' },
@@ -146,6 +152,7 @@ static struct option longopts[] =
 	{ "passive-portrange-default", no_argument, NULL, 'T' },
 	{ "verbose", no_argument, NULL, 'v' },
 	{ "retry-delay", required_argument, NULL, 'w' },
+	{ "request", required_argument, NULL, 'X' },
 
 	/* options without a single character equivalent */
 	{ "bind-address", required_argument, NULL, OPTION_BIND_ADDRESS },
@@ -418,11 +425,23 @@ query_auth(struct url *URL)
 	return (0);
 }
 
+/*
+ * Decide how to invoke libfetch
+ */
+static FILE *
+invoke(struct url *url, struct url_stat *us, const char *flags, bool is_http)
+{
+	if (is_http)
+		return (fetchXReqHTTP(url, us, method ? method : "GET", flags,
+		    &headers, NULL, NULL));
+	return (fetchXGet(url, us, flags));
+}
+
 /*
  * Fetch a file
  */
 static int
-fetch(char *URL, const char *path, int *is_http)
+fetch(char *URL, const char *path, bool *is_http)
 {
 	struct url *url;
 	struct url_stat us;
@@ -576,7 +595,7 @@ fetch(char *URL, const char *path, int *is_http)
 	/* start the transfer */
 	if (timeout)
 		alarm(timeout);
-	f = fetchXGet(url, &us, flags);
+	f = invoke(url, &us, flags, *is_http);
 	if (timeout)
 		alarm(0);
 	if (sigalrm || sigint)
@@ -708,7 +727,7 @@ fetch(char *URL, const char *path, int *is_http)
 			 * from scratch if we want the whole file
 			 */
 			url->offset = 0;
-			if ((f = fetchXGet(url, &us, flags)) == NULL) {
+			if ((f = invoke(url, &us, flags, *is_http)) == NULL) {
 				warnx("%s: %s", URL, fetchLastErrString);
 				goto failure;
 			}
@@ -883,22 +902,40 @@ fetch(char *URL, const char *path, int *is_http)
 	return (r);
 }
 
+static void
+add_header(char *arg)
+{
+	struct http_header *hdr;
+	char *split;
+
+	if ((split = strchr(arg, ':')) == NULL)
+		errx(1, "invalid header (%s)", arg);
+	*split = '\0';
+	if ((hdr = malloc(sizeof(*hdr))) == NULL)
+		errx(1, "%s", strerror(ENOMEM));
+	hdr->name = arg;
+	hdr->value = split + 1;
+	hdr->value += strspn(hdr->value, " \t"); /* skip leading whitespace */
+	TAILQ_INSERT_TAIL(&headers, hdr, headers);
+}
+
 static void
 usage(void)
 {
-	fprintf(stderr, "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n",
+	fprintf(stderr, "%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n",
 "usage: fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [--bind-address=host]",
 "       [--ca-cert=file] [--ca-path=dir] [--cert=file] [--crl=file]",
-"       [-i file] [--key=file] [-N file] [--no-passive] [--no-proxy=list]",
-"       [--no-sslv3] [--no-tlsv1] [--no-verify-hostname] [--no-verify-peer]",
-"       [-o file] [--referer=URL] [-S bytes] [-T seconds]",
-"       [--user-agent=agent-string] [-w seconds] URL ...",
+"       [-H header] [-i file] [--key=file] [-N file] [--no-passive]",
+"       [--no-proxy=list] [--no-sslv3] [--no-tlsv1] [--no-verify-hostname]",
+"       [--no-verify-peer] [-o file] [--referer=URL] [-S bytes] [-T seconds]",
+"       [--user-agent=agent-string] [-w seconds] [-X method] URL ...",
 "       fetch [-146AadFlMmnPpqRrsUv] [-B bytes] [--bind-address=host]",
 "       [--ca-cert=file] [--ca-path=dir] [--cert=file] [--crl=file]",
-"       [-i file] [--key=file] [-N file] [--no-passive] [--no-proxy=list]",
-"       [--no-sslv3] [--no-tlsv1] [--no-verify-hostname] [--no-verify-peer]",
-"       [-o file] [--referer=URL] [-S bytes] [-T seconds]",
-"       [--user-agent=agent-string] [-w seconds] -h host -f file [-c dir]");
+"       [-H header] [-i file] [--key=file] [-N file] [--no-passive]",
+"       [--no-proxy=list] [--no-sslv3] [--no-tlsv1] [--no-verify-hostname]",
+"       [--no-verify-peer] [-o file] [--referer=URL] [-S bytes] [-T seconds]",
+"       [--user-agent=agent-string] [-w seconds] [-X method] -h host -f file",
+"       [-c dir]");
 }
 
 
@@ -912,11 +949,12 @@ main(int argc, char *argv[])
 	struct sigaction sa;
 	const char *p, *s;
 	char *end, *q;
-	int c, e, is_http, r;
+	int c, e, r;
+	bool is_http;
 
 
 	while ((c = getopt_long(argc, argv,
-	    "146AaB:bc:dFf:Hh:i:lMmN:nPpo:qRrS:sT:tUvw:",
+	    "146AaB:bc:dFf:H:h:i:lMmN:nPpo:qRrS:sT:tUvw:X:",
 	    longopts, NULL)) != -1)
 		switch (c) {
 		case '1':
@@ -956,8 +994,7 @@ main(int argc, char *argv[])
 			f_filename = optarg;
 			break;
 		case 'H':
-			warnx("the -H option is now implicit, "
-			    "use -U to disable");
+			add_header(optarg);
 			break;
 		case 'h':
 			h_hostname = optarg;
@@ -1031,6 +1068,9 @@ main(int argc, char *argv[])
 			if (*optarg == '\0' || *end != '\0')
 				errx(1, "invalid delay (%s)", optarg);
 			break;
+		case 'X':
+			method = optarg;
+			break;
 		case OPTION_BIND_ADDRESS:
 			setenv("FETCH_BIND_ADDRESS", optarg, 1);
 			break;