support fly's tls-offload for doh and dot

README.md

@@ -33,17 +33,27 @@ curl abcxyz.neverssl.com --resolve 'abcxyz.neverssl.com:80:<midway-ip>' -v
 ```
 
 ### DNS
-midway runs DoT and DoH stub resolver on ports 443 and 853 (or 8443 and 8853 in
-non-previledge mode), forwarding queries to `UPSTREAM_DOH` env var (The Google
-Public Resolver is the default). For TLS termination, Cert/Key pair can be ethier
+*Midway* runs Fly.io-terminated TLS DoT and DoH stub resolvers on ports `1443` and `1853`,
+this essentially means, you can access DoT over `<your-app-name>.fly.dev:1853` and DoH
+over `https://<your-app-name>.fly.dev:1443`.
+
+*Midway* also runs DoT and DoH stub resolver on ports `443` and `853` (or `8443` and
+`8853` in non-previledge mode). For TLS termination, Cert/Key pair can be ethier
 supplied by setting env vars, `TLS_CERT_PATH` / `TLS_KEY_PATH` pointing to cert
-/ key files, or by base64 encoding the contents of the file into env var,
+/ key files; or by base64 encoding the contents of the file into env var,
 `TLS_CERTKEY` like so:
 
 ```bash
-TLS_CERTKEY = "KEY=b64(key-contents)\nCRT=b64(cert-contents)"
+# either
+TLS_KEY_PATH = "/path/to/pem.key"
+TLS_CERT_PATH = "/path/to/pem.cert"
+# or
+TLS_CERTKEY = "KEY=b64(key-pem-contents)\nCRT=b64(cert-pem-contents)"
 ```
 
+The stub-resovler forwards queries to `UPSTREAM_DOH` env var (The Google
+DoH public resolver `https://dns.google/dns-query` is the default).
+
 Test certs for DNS over TLS and DNS over HTTPS in `/test/certs/` are generated
 via openssl ([ref](https://github.com/denji/golang-tls)).
 

fly.toml

@@ -40,9 +40,9 @@ processes = []
     handlers = ["proxy_proto"]
     port = "5001"
 
-# proxifier and doh svc on 80, 443
+# h1x on 80
 [[services]]
-  internal_port = 443
+  internal_port = 80
   protocol = "tcp"
 
   [services.concurrency]
@@ -58,10 +58,11 @@ processes = []
 
   [[services.ports]]
     handlers = ["proxy_proto"]
-    port = "443"
+    port = "80"
 
+# proxifier and doh svc on 80, 443
 [[services]]
-  internal_port = 80
+  internal_port = 443
   protocol = "tcp"
 
   [services.concurrency]
@@ -77,7 +78,7 @@ processes = []
 
   [[services.ports]]
     handlers = ["proxy_proto"]
-    port = "80"
+    port = "443"
 
 # dot svc on port 853
 [[services]]
@@ -98,3 +99,46 @@ processes = []
   [[services.ports]]
     handlers = ["proxy_proto"]
     port = "853"
+
+# doh with fly-terminated tls on 1443
+# community.fly.io/t/4449
+[[services]]
+  internal_port = 1443
+  protocol = "tcp"
+
+  [services.concurrency]
+  hard_limit = 512
+  soft_limit = 256
+  type = "connections"
+
+  [[services.tcp_checks]]
+  grace_period = "5s"
+  interval = "30s"
+  restart_limit = 6
+  timeout = "3s"
+
+  [[services.ports]]
+    handlers = ["tls", "proxy_proto"]
+    tls_options = { alpn = ["h2", "http/1.1"] }
+    port = "1443"
+
+# dot with fly-terminated tls on port 1853
+# community.fly.io/t/4495
+[[services]]
+  internal_port = 1853
+  protocol = "tcp"
+
+  [services.concurrency]
+  hard_limit = 512
+  soft_limit = 256
+  type = "connections"
+
+  [[services.tcp_checks]]
+  grace_period = "5s"
+  interval = "30s"
+  restart_limit = 6
+  timeout = "3s"
+
+  [[services.ports]]
+    handlers = ["tls", "proxy_proto"]
+    port = "1853"

go.mod

@@ -5,9 +5,10 @@ go 1.18
 require (
 	github.com/miekg/dns v1.1.48
 	github.com/pires/go-proxyproto v0.6.2
+	golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985
 )
 
 require (
-	golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 // indirect
 	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
+	golang.org/x/text v0.3.6 // indirect
 )

go.sum

@@ -24,6 +24,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=

main.go

@@ -15,6 +15,10 @@ import (
 	"sync"
 	"time"
 
+	"golang.org/x/net/http2"
+
+	"golang.org/x/net/http2/h2c"
+
 	"github.com/miekg/dns"
 	proxyproto "github.com/pires/go-proxyproto"
 )
@@ -29,12 +33,12 @@ var (
 
 // Adopted from: github.com/inetaf/tcpproxy/blob/be3ee21/tcpproxy.go
 func main() {
-	totallisteners := 6
-
 	portmap := map[string]string{
+		"h11":    ":80",
 		"tls":    ":443",
 		"dot":    ":853",
-		"h11":    ":80",
+		"flydoh": ":1443",
+		"flydot": ":1853",
 		"echo":   ":5000",
 		"ppecho": ":5001",
 	}
@@ -44,28 +48,48 @@ func main() {
 		portmap["h11"] = ":8080"
 	}
 
+	totallisteners := len(portmap)
 	hold := barrier(totallisteners)
 
+	// cleartext http1.x on port 80
+	t80, err := net.Listen("tcp", portmap["h11"])
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println("started: pptcp-server on port ", portmap["h11"])
+	pp80 := &proxyproto.Listener{Listener: t80}
+
+	// tcp-tls (http2 / http1.1) on port 443
 	t443, err := net.Listen("tcp", portmap["tls"])
 	if err != nil {
 		log.Fatal(err)
 	}
 	fmt.Println("started: pptcp-server on port ", portmap["tls"])
 	pp443 := &proxyproto.Listener{Listener: t443}
 
+	// tcp-tls (DNS over TLS) on port 853
 	t853, err := net.Listen("tcp", portmap["dot"])
 	if err != nil {
 		log.Fatal(err)
 	}
 	fmt.Println("started: pptcp-server on port ", portmap["dot"])
 	pp853 := &proxyproto.Listener{Listener: t853}
 
-	t80, err := net.Listen("tcp", portmap["h11"])
+	// fly terminated tls (http2 and http1.1) on port 1443
+	t1443, err := net.Listen("tcp", portmap["flydoh"])
 	if err != nil {
 		log.Fatal(err)
 	}
-	fmt.Println("started: pptcp-server on port ", portmap["h11"])
-	pp80 := &proxyproto.Listener{Listener: t80}
+	fmt.Println("started: pptcp-server on port ", portmap["flydoh"])
+	pp1443 := &proxyproto.Listener{Listener: t1443}
+
+	// fly terminated tls (DNS over TLS) on port 1853
+	t1853, err := net.Listen("tcp", portmap["flydot"])
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println("started: pptcp-server on port ", portmap["flydot"])
+	pp1853 := &proxyproto.Listener{Listener: t1853}
 
 	// ref: fly.io/docs/app-guides/udp-and-tcp/
 	u5000, err := net.ListenPacket("udp", "fly-global-services:5000")
@@ -100,14 +124,16 @@ func main() {
 		Transport: tr,
 	}
 
-	go echoUDP(u5000, hold)
-	go echoTCP(t5000, hold)
-	go echoPP(pp5001, hold)
 	// proxyproto listener works with plain tcp, too
+	go startPP(pp80, hold)
 	go startPPWithDoH(pp443, dohresolver, hold)
 	go startPPWithDoT(pp853, dohresolver, hold)
-	go startPP(pp80, hold)
-
+	go startPPWithDoHCleartext(pp1443, dohresolver, hold)
+	go startPPWithDoTCleartext(pp1853, dohresolver, hold)
+	// echo servers on tcp and udp
+	go echoUDP(u5000, hold)
+	go echoTCP(t5000, hold)
+	go echoPP(pp5001, hold)
 	hold.Wait()
 }
 
@@ -210,6 +236,59 @@ func startPP(tcp *proxyproto.Listener, wg *sync.WaitGroup) {
 	}
 }
 
+// ref: github.com/thrawn01/h2c-golang-example
+func startPPWithDoHCleartext(tcp *proxyproto.Listener, doh *http.Client, wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	if tcp == nil {
+		log.Print("Exiting pp doh")
+		return
+	}
+
+	log.Print("mode: DoH cleartext ", tcp.Addr().String())
+
+	h2s := &http2.Server{}
+	doh2c := http.HandlerFunc(dohHandler(doh))
+	hello := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "Hello, %v, http: %v", r.URL.Path, r.TLS == nil)
+	})
+
+	mux := http.NewServeMux()
+	mux.Handle("/h/w", h2c.NewHandler(hello, h2s))
+	mux.Handle("/", h2c.NewHandler(doh2c, h2s))
+	dnsserver := &http.Server{
+		Handler:      mux,
+		ReadTimeout:  conntimeout,
+		WriteTimeout: conntimeout,
+	}
+
+	err := dnsserver.Serve(tcp)
+	log.Print("exit doh cleartext:", err)
+}
+
+func startPPWithDoTCleartext(tcp *proxyproto.Listener, doh *http.Client, wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	if tcp == nil {
+		log.Print("Exiting pp dot")
+		return
+	}
+
+	log.Print("mode: DoT cleartext ", tcp.Addr().String())
+	// ref: github.com/miekg/dns/blob/dedee46/server.go#L192
+	// usage: github.com/folbricht/routedns/blob/7b8e284/dotlistener.go#L29
+	dnsserver := &dns.Server{
+		Net:           "tcp", // unused
+		Listener:      tcp,
+		MaxTCPQueries: int(maxInflightQueries),
+		Handler:       dnsHandler(doh),
+	}
+
+	// ref: github.com/miekg/dns/blob/dedee46/server.go#L133
+	err := dnsserver.ActivateAndServe()
+	log.Print("exit dot cleartext:", err)
+}
+
 func barrier(count int) *sync.WaitGroup {
 	w := &sync.WaitGroup{}
 	w.Add(count)