Add reqtest, reqhtml; deprecate requests.ToHTML and testing transports (#115)

* Add reqtest, reqhtml; deprecate requests.ToHTML and testing transports

* reqtest: Add package doc

* reqtest: Rename to reqtest.Server and improve docs

* reqhtml.Body: Better doc string

* Docs: Better docs for reqtest.Replay.

Author: earthboundkid

README.md

@@ -225,14 +225,14 @@ fmt.Println(u.String()) // https://dev1.example.com/get?a=1&b=3&c=4
 // record a request to the file system
 var s1, s2 string
 err := requests.URL("http://example.com").
-	Transport(requests.Record(nil, "somedir")).
+	Transport(reqtest.Record(nil, "somedir")).
 	ToString(&s1).
 	Fetch(ctx)
 check(err)
 
 // now replay the request in tests
 err = requests.URL("http://example.com").
-	Transport(requests.Replay("somedir")).
+	Transport(reqtest.Replay("somedir")).
 	ToString(&s2).
 	Fetch(ctx)
 check(err)

config.go

@@ -34,6 +34,8 @@ func GzipConfig(level int, h func(gw *gzip.Writer) error) Config {
 // TestServerConfig returns a Config
 // which sets the Builder's BaseURL to s.URL
 // and the Builder's Client to s.Client().
+//
+// Deprecated: Use reqtest.Server.
 func TestServerConfig(s *httptest.Server) Config {
 	return func(rb *Builder) {
 		rb.

handler.go

@@ -94,6 +94,8 @@ func ToBufioScanner(f func(r *bufio.Scanner) error) ResponseHandler {
 }
 
 // ToHTML parses the page with x/net/html.Parse.
+//
+// Deprecated: Use reqhtml.To.
 func ToHTML(n *html.Node) ResponseHandler {
 	return ToBufioReader(func(r *bufio.Reader) error {
 		n2, err := html.Parse(r)

recorder.go

@@ -18,6 +18,8 @@ import (
 // requests and their responses to text files in basepath.
 // Requests are named according to a hash of their contents.
 // Responses are named according to the request that made them.
+//
+// Deprecated: Use reqtest.Record.
 func Record(rt http.RoundTripper, basepath string) Transport {
 	if rt == nil {
 		rt = http.DefaultTransport
@@ -56,6 +58,8 @@ func Record(rt http.RoundTripper, basepath string) Transport {
 // Replay returns an http.RoundTripper that reads its
 // responses from text files in basepath.
 // Responses are looked up according to a hash of the request.
+//
+// Deprecated: Use reqtest.Replay.
 func Replay(basepath string) Transport {
 	return ReplayFS(os.DirFS(basepath))
 }
@@ -66,6 +70,8 @@ var errNotFound = errors.New("response not found")
 // responses from text files in the fs.FS.
 // Responses are looked up according to a hash of the request.
 // Response file names may optionally be prefixed with comments for better human organization.
+//
+// Deprecated: Use reqtest.ReplayFS.
 func ReplayFS(fsys fs.FS) Transport {
 	return RoundTripFunc(func(req *http.Request) (res *http.Response, err error) {
 		defer func() {
@@ -110,6 +116,8 @@ func buildName(b []byte) (reqname, resname string) {
 // it caches the result of issuing the request with rt in basepath.
 // Requests are named according to a hash of their contents.
 // Responses are named according to the request that made them.
+//
+// Deprecated: Use reqtest.Caching.
 func Caching(rt http.RoundTripper, basepath string) Transport {
 	replay := Replay(basepath).RoundTrip
 	record := Record(rt, basepath).RoundTrip

recorder_test.go

@@ -13,11 +13,14 @@ import (
 )
 
 func TestRecordReplay(t *testing.T) {
+	baseTrans := requests.ReplayString(`HTTP/1.1 200 OK
+
+Test Document 1`)
 	dir := t.TempDir()
 
 	var s1, s2 string
 	err := requests.URL("http://example.com").
-		Transport(requests.Record(http.DefaultTransport, dir)).
+		Transport(requests.Record(baseTrans, dir)).
 		ToString(&s1).
 		Fetch(context.Background())
 	be.NilErr(t, err)
@@ -28,6 +31,7 @@ func TestRecordReplay(t *testing.T) {
 		Fetch(context.Background())
 	be.NilErr(t, err)
 	be.Equal(t, s1, s2)
+	be.Equal(t, "Test Document 1", s1)
 }
 
 func TestCaching(t *testing.T) {

reqhtml/html.go

@@ -0,0 +1,27 @@
+// Package reqhtml contains utilities for sending and receiving x/net/html objects.
+package reqhtml
+
+import (
+	"io"
+
+	"github.com/carlmjohnson/requests"
+	"golang.org/x/net/html"
+)
+
+// To decodes a response as an html document.
+func To(n *html.Node) requests.ResponseHandler {
+	return requests.ToHTML(n)
+}
+
+// Body sets the requests.Builder's request body to the HTML document.
+// It also sets ContentType to "text/html"
+// if it is not otherwise set.
+func Body(n *html.Node) requests.Config {
+	return func(rb *requests.Builder) {
+		rb.
+			Body(requests.BodyWriter(func(w io.Writer) error {
+				return html.Render(w, n)
+			})).
+			HeaderOptional("context-type", "text/html")
+	}
+}

reqhtml/html_test.go

@@ -0,0 +1,75 @@
+package reqhtml_test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httputil"
+
+	"github.com/carlmjohnson/requests"
+	"github.com/carlmjohnson/requests/reqhtml"
+	"github.com/carlmjohnson/requests/reqtest"
+	"golang.org/x/net/html"
+	"golang.org/x/net/html/atom"
+)
+
+func init() {
+	http.DefaultTransport = reqtest.ReplayString(`HTTP/1.1 200 OK
+
+	<a href="https://www.iana.org/domains/example"></a>`)
+}
+
+func ExampleTo() {
+	var doc html.Node
+	err := requests.
+		URL("http://example.com").
+		Handle(reqhtml.To(&doc)).
+		Fetch(context.Background())
+	if err != nil {
+		fmt.Println("could not connect to example.com:", err)
+	}
+	var f func(*html.Node)
+	f = func(n *html.Node) {
+		if n.DataAtom == atom.A {
+			for _, attr := range n.Attr {
+				if attr.Key == "href" {
+					fmt.Println("link:", attr.Val)
+				}
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			f(c)
+		}
+	}
+	f(&doc)
+	// Output:
+	// link: https://www.iana.org/domains/example
+}
+
+func ExampleBody() {
+	link := html.Node{
+		Type: html.ElementNode,
+		Data: "a",
+		Attr: []html.Attribute{
+			{Key: "href", Val: "http://example.com"},
+		},
+	}
+	text := html.Node{
+		Type: html.TextNode,
+		Data: "Hello, World!",
+	}
+	link.AppendChild(&text)
+
+	req, err := requests.
+		URL("http://example.com").
+		Config(reqhtml.Body(&link)).
+		Request(context.Background())
+	b, err := httputil.DumpRequest(req, true)
+	if err != nil {
+		panic(err)
+	}
+	fmt.Printf("%q\n", b)
+
+	// Output:
+	// "POST / HTTP/1.1\r\nHost: example.com\r\nContext-Type: text/html\r\n\r\n<a href=\"http://example.com\">Hello, World!</a>"
+}

reqtest/doc.go

@@ -0,0 +1,2 @@
+// Package reqtest contains helpers for writing tests of HTTP clients and servers.
+package reqtest

reqtest/recorder_example_test.go

@@ -0,0 +1,53 @@
+package reqtest_test
+
+import (
+	"context"
+	"fmt"
+	"testing/fstest"
+
+	"github.com/carlmjohnson/requests"
+	"github.com/carlmjohnson/requests/reqtest"
+)
+
+func ExampleReplayString() {
+	const res = `HTTP/1.1 200 OK
+
+An example response.`
+
+	var s string
+	const expected = `An example response.`
+	if err := requests.
+		URL("http://response.example").
+		Transport(reqtest.ReplayString(res)).
+		ToString(&s).
+		Fetch(context.Background()); err != nil {
+		panic(err)
+	}
+	fmt.Println(s == expected)
+	// Output:
+	// true
+}
+
+func ExampleReplayFS() {
+	fsys := fstest.MapFS{
+		"fsys.example - MKIYDwjs.res.txt": &fstest.MapFile{
+			Data: []byte(`HTTP/1.1 200 OK
+Content-Type: text/plain; charset=UTF-8
+Date: Mon, 24 May 2021 18:48:50 GMT
+
+An example response.`),
+		},
+	}
+	var s string
+	const expected = `An example response.`
+	if err := requests.
+		URL("http://fsys.example").
+		Transport(reqtest.ReplayFS(fsys)).
+		ToString(&s).
+		Fetch(context.Background()); err != nil {
+		panic(err)
+	}
+	fmt.Println(s == expected)
+	// Output:
+	// true
+}

reqtest/recorder_test.go

@@ -0,0 +1,69 @@
+package reqtest_test
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/carlmjohnson/requests"
+	"github.com/carlmjohnson/requests/internal/be"
+	"github.com/carlmjohnson/requests/reqtest"
+)
+
+func TestRecordReplay(t *testing.T) {
+	baseTrans := requests.ReplayString(`HTTP/1.1 200 OK
+
+Test Document 1`)
+	dir := t.TempDir()
+
+	var s1, s2 string
+	err := requests.URL("http://example.com").
+		Transport(reqtest.Record(baseTrans, dir)).
+		ToString(&s1).
+		Fetch(context.Background())
+	be.NilErr(t, err)
+
+	err = requests.URL("http://example.com").
+		Transport(reqtest.Replay(dir)).
+		ToString(&s2).
+		Fetch(context.Background())
+	be.NilErr(t, err)
+	be.Equal(t, s1, s2)
+	be.Equal(t, "Test Document 1", s1)
+}
+
+func TestCaching(t *testing.T) {
+	dir := t.TempDir()
+	hasRun := false
+	content := "some content"
+	var onceTrans requests.RoundTripFunc = func(req *http.Request) (res *http.Response, err error) {
+		be.False(t, hasRun)
+		hasRun = true
+		res = &http.Response{
+			StatusCode: http.StatusOK,
+			Body:       io.NopCloser(strings.NewReader(content)),
+		}
+		return
+	}
+	trans := reqtest.Caching(onceTrans, dir)
+	var s1, s2 string
+	err := requests.URL("http://example.com").
+		Transport(trans).
+		ToString(&s1).
+		Fetch(context.Background())
+	be.NilErr(t, err)
+	err = requests.URL("http://example.com").
+		Transport(trans).
+		ToString(&s2).
+		Fetch(context.Background())
+	be.NilErr(t, err)
+	be.Equal(t, content, s1)
+	be.Equal(t, s1, s2)
+
+	entries, err := os.ReadDir(dir)
+	be.NilErr(t, err)
+	be.Equal(t, 2, len(entries))
+}

reqtest/server.go

@@ -0,0 +1,14 @@
+package reqtest
+
+import (
+	"net/http/httptest"
+
+	"github.com/carlmjohnson/requests"
+)
+
+// Server takes an httptest.Server and returns a requests.Config
+// which sets the requests.Builder's BaseURL to s.URL
+// and the requests.Builder's Client to s.Client().
+func Server(s *httptest.Server) requests.Config {
+	return requests.TestServerConfig(s)
+}

reqtest/server_example_test.go

@@ -0,0 +1,54 @@
+package reqtest_test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+
+	"github.com/carlmjohnson/requests"
+	"github.com/carlmjohnson/requests/reqtest"
+)
+
+func ExampleServer() {
+	// Create an httptest.Server for your project's router
+	mux := http.NewServeMux()
+	mux.HandleFunc("/greeting", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "Hello, world!")
+	})
+	mux.HandleFunc("/salutation", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "Howdy, planet!")
+	})
+
+	srv := httptest.NewServer(mux)
+	defer srv.Close()
+
+	// Now test that the handler has the expected return values
+	{
+		var s string
+		err := requests.
+			New(reqtest.Server(srv)).
+			Path("/greeting").
+			ToString(&s).
+			Fetch(context.Background())
+		if err != nil {
+			fmt.Println("Error!", err)
+		}
+		fmt.Println(s) // Hello, world!
+	}
+	{
+		var s string
+		err := requests.
+			New(reqtest.Server(srv)).
+			Path("/salutation").
+			ToString(&s).
+			Fetch(context.Background())
+		if err != nil {
+			fmt.Println("Error!", err)
+		}
+		fmt.Println(s) // Howdy, planet!
+	}
+	// Output:
+	// Hello, world!
+	// Howdy, planet!
+}