lib/libfetch: Add fetchXReqHTTP(3)

Modern APIs often rely on HTTP headers as part of their interface, both
in requests and responses.

Add a function to libfetch allowing additional request headers to be
specified for an HTTP request and response headers to be returned to the
caller.

This function accepts an optional stream for the body content, as a
nul-terminated C string may be inadequate for some content.  If the
length of the stream can be determined, a Content-Length header is
added.  Otherwise, the body is sent chunk-encoded.

Handling of HTTP trailers remains unimplemented.

Signed-off-by: Ryan Moeller <[email protected]>

Author: ryan-moeller

lib/libfetch/Makefile

@@ -48,6 +48,7 @@ httperr.h: http.errors ${.CURDIR}/Makefile
 	@echo "    { -1, FETCH_UNKNOWN, \"Unknown HTTP error\" }" >> ${.TARGET}
 	@echo "};" >> ${.TARGET}
 
+MLINKS+= fetch.3 fetchFreeResHeaders.3
 MLINKS+= fetch.3 fetchFreeURL.3
 MLINKS+= fetch.3 fetchGet.3
 MLINKS+= fetch.3 fetchGetFTP.3
@@ -77,5 +78,6 @@ MLINKS+= fetch.3 fetchXGetFTP.3
 MLINKS+= fetch.3 fetchXGetFile.3
 MLINKS+= fetch.3 fetchXGetHTTP.3
 MLINKS+= fetch.3 fetchXGetURL.3
+MLINKS+= fetch.3 fetchXReqHTTP.3
 
 .include <bsd.lib.mk>

lib/libfetch/common.h

@@ -161,9 +161,6 @@ int		 fetch_no_proxy_match(const char *);
  */
 FILE		*http_request(struct url *, const char *,
 		     struct url_stat *, struct url *, const char *);
-FILE		*http_request_body(struct url *, const char *,
-		     struct url_stat *, struct url *, const char *,
-		     const char *, const char *);
 FILE		*ftp_request(struct url *, const char *,
 		     struct url_stat *, struct url *, const char *);
 

lib/libfetch/fetch.3

@@ -24,7 +24,7 @@
 .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
-.Dd October 7, 2023
+.Dd September 27, 2025
 .Dt FETCH 3
 .Os
 .Sh NAME
@@ -52,6 +52,8 @@
 .Nm fetchStatHTTP ,
 .Nm fetchListHTTP ,
 .Nm fetchReqHTTP ,
+.Nm fetchXReqHTTP ,
+.Nm fetchFreeResHeaders ,
 .Nm fetchXGetFTP ,
 .Nm fetchGetFTP ,
 .Nm fetchPutFTP ,
@@ -112,6 +114,11 @@
 .Fn fetchListHTTP "struct url *u" "const char *flags"
 .Ft FILE *
 .Fn fetchReqHTTP "struct url *u" "const char *method" "const char *flags" "const char *content_type" "const char *body"
+.In sys/queue.h
+.Ft FILE *
+.Fn fetchXReqHTTP "struct url *u" "struct url_stat *us" "const char *method" "const char *flags" "const struct http_headers *reqhdrs" "struct http_headers *reshdrs" "FILE *body"
+.Ft void
+.Fn fetchFreeResHeaders "struct http_headers *reshdrs"
 .Ft FILE *
 .Fn fetchXGetFTP "struct url *u" "struct url_stat *us" "const char *flags"
 .Ft FILE *
@@ -357,9 +364,10 @@ and password "anonymous@<hostname>".
 The
 .Fn fetchXGetHTTP ,
 .Fn fetchGetHTTP ,
-.Fn fetchPutHTTP
-and
+.Fn fetchPutHTTP ,
 .Fn fetchReqHTTP
+and
+.Fn fetchXReqHTTP
 functions implement the HTTP/1.1 protocol.
 With a little luck, there is
 even a chance that they comply with RFC2616 and RFC2617.
@@ -389,8 +397,10 @@ will send a conditional
 HTTP header to only fetch the content if it is newer than
 .Va ims_time .
 .Pp
-The function
+The functions
 .Fn fetchReqHTTP
+and
+.Fn fetchXReqHTTP
 can be used to make requests with an arbitrary HTTP verb,
 including POST, DELETE, CONNECT, OPTIONS, TRACE or PATCH.
 This can be done by setting the argument
@@ -401,10 +411,34 @@ and
 .Fa body
 to the content.
 .Pp
-Since there seems to be no good way of implementing the HTTP PUT
-method in a manner consistent with the rest of the
-.Nm fetch
-library,
+In addition to returning
+.Vt "struct url_stat"
+information,
+.Fn fetchXReqHTTP
+also accepts an optional list of arbitrary additional headers to send before the
+body,
+an optional list to be filled with all response headers received,
+and
+an optional
+.Vt "FILE"
+stream to send as the body.
+The response headers list should be freed with
+.Fn fetchFreeResHeaders .
+The
+.Vt http_header
+and
+.Vt http_headers
+structures are defined as follows in
+.In fetch.h :
+.Bd -literal
+struct http_header {
+    const char *name;
+    const char *value;
+    TAILQ_ENTRY(http_header) headers;
+};
+TAILQ_HEAD(http_headers, http_header);
+.Ed
+.Pp
 .Fn fetchPutHTTP
 is currently unimplemented.
 .Sh HTTPS SCHEME

lib/libfetch/fetch.h

@@ -31,7 +31,7 @@
 #ifndef _FETCH_H_INCLUDED
 #define _FETCH_H_INCLUDED
 
-#define _LIBFETCH_VER "libfetch/2.0"
+#define _LIBFETCH_VER "libfetch/2.1"
 
 #define URL_SCHEMELEN 16
 #define URL_USERLEN 256
@@ -61,6 +61,15 @@ struct url_ent {
 	struct url_stat	 stat;
 };
 
+#ifdef _SYS_QUEUE_H_
+struct http_header {
+	const char	*name;
+	const char	*value;
+	TAILQ_ENTRY(http_header)	headers;
+};
+TAILQ_HEAD(http_headers, http_header);
+#endif
+
 /* Recognized schemes */
 #define SCHEME_FTP	"ftp"
 #define SCHEME_HTTP	"http"
@@ -105,6 +114,12 @@ int		 fetchStatHTTP(struct url *, struct url_stat *, const char *);
 struct url_ent	*fetchListHTTP(struct url *, const char *);
 FILE		*fetchReqHTTP(struct url *, const char *, const char *,
 		    const char *, const char *);
+#ifdef _SYS_QUEUE_H_
+FILE		*fetchXReqHTTP(struct url *, struct url_stat *, const char *,
+		    const char *, const struct http_headers *,
+		    struct http_headers *, FILE *);
+void		 fetchFreeResHeaders(struct http_headers *);
+#endif
 
 /* FTP-specific functions */
 FILE		*fetchXGetFTP(struct url *, struct url_stat *, const char *);