From 476478ba0aa367046c840dc6dcea8a9204d5ed17 Mon Sep 17 00:00:00 2001 From: Seo Suchan Date: Thu, 16 Feb 2023 18:31:37 +0900 Subject: [PATCH 01/10] dpi solver for http using nfqueue --- cmd/flags.go | 4 + cmd/setup_challenges.go | 7 ++ go.mod | 10 ++ go.sum | 16 +++ providers/http/nfqueue/nfqueue.go | 196 ++++++++++++++++++++++++++++++ 5 files changed, 233 insertions(+) create mode 100644 providers/http/nfqueue/nfqueue.go diff --git a/cmd/flags.go b/cmd/flags.go index 902d3526ea..4ec371e8cb 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -85,6 +85,10 @@ func CreateFlags(defaultPath string) []cli.Flag { Name: "http.memcached-host", Usage: "Set the memcached host(s) to use for HTTP based challenges. Challenges will be written to all specified hosts.", }, + &cli.StringFlag{ + Name: "http.nfqueueport", + Usage: "Set the port to use for HTTP based challange. but unlike http it will not bind that port and while other thing already binding that port.", + }, &cli.BoolFlag{ Name: "tls", Usage: "Use the TLS challenge to solve challenges. Can be mixed with other types of challenges.", diff --git a/cmd/setup_challenges.go b/cmd/setup_challenges.go index 938ee74592..469a062ad8 100644 --- a/cmd/setup_challenges.go +++ b/cmd/setup_challenges.go @@ -13,6 +13,7 @@ import ( "github.com/go-acme/lego/v4/log" "github.com/go-acme/lego/v4/providers/dns" "github.com/go-acme/lego/v4/providers/http/memcached" + nfqueue "github.com/go-acme/lego/v4/providers/http/nfqueue" "github.com/go-acme/lego/v4/providers/http/webroot" "github.com/urfave/cli/v2" ) @@ -55,6 +56,12 @@ func setupHTTPProvider(ctx *cli.Context) challenge.Provider { log.Fatal(err) } return ps + case ctx.IsSet("http.nfqueueport"): + ps, err := nfqueue.NewHttpDpiProvider(ctx.String("http.nfqueueport")) + if err != nil { + log.Fatal(err) + } + return ps case ctx.IsSet("http.port"): iface := ctx.String("http.port") if !strings.Contains(iface, ":") { diff --git a/go.mod b/go.mod index cb2425ee98..bf66b9077c 100644 --- a/go.mod +++ b/go.mod @@ -71,6 +71,11 @@ require ( software.sslmate.com/src/go-pkcs12 v0.2.0 ) +require ( + github.com/florianl/go-nfqueue v1.3.1 + github.com/google/gopacket v1.1.19 +) + require ( github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect @@ -92,18 +97,22 @@ require ( github.com/golang-jwt/jwt/v4 v4.2.0 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.8 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/native v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b // indirect github.com/labbsr0x/goh v1.0.1 // indirect github.com/liquidweb/go-lwApi v0.0.5 // indirect github.com/liquidweb/liquidweb-cli v0.6.9 // indirect + github.com/mdlayher/netlink v1.6.0 // indirect + github.com/mdlayher/socket v0.1.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -123,6 +132,7 @@ require ( go.opencensus.io v0.22.3 // indirect go.uber.org/ratelimit v0.2.0 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect golang.org/x/sys v0.4.0 // indirect golang.org/x/text v0.6.0 // indirect golang.org/x/tools v0.1.12 // indirect diff --git a/go.sum b/go.sum index 293d747509..5cf94f685a 100644 --- a/go.sum +++ b/go.sum @@ -132,6 +132,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/florianl/go-nfqueue v1.3.1 h1:khQ9fYCrjbu5CF8dZF55G2RTIEIQRI0Aj5k3msJR6Gw= +github.com/florianl/go-nfqueue v1.3.1/go.mod h1:aHWbgkhryJxF5XxYvJ3oRZpdD4JP74Zu/hP1zuhja+M= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= @@ -215,13 +217,17 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -300,6 +306,8 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/josharian/native v1.0.0 h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTxk= +github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -373,6 +381,10 @@ github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mdlayher/netlink v1.6.0 h1:rOHX5yl7qnlpiVkFWoqccueppMtXzeziFjWAjLg6sz0= +github.com/mdlayher/netlink v1.6.0/go.mod h1:0o3PlBmGst1xve7wQ7j/hwpNaFaH4qCRyWCdcZk8/vA= +github.com/mdlayher/socket v0.1.1 h1:q3uOGirUPfAV2MUoaC7BavjQ154J7+JOkTWyiV+intI= +github.com/mdlayher/socket v0.1.1/go.mod h1:mYV5YIZAfHh4dzDVzI8x8tWLWCliuX8Mon5Awbj+qDs= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.47/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= @@ -675,7 +687,9 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= @@ -696,6 +710,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -745,6 +760,7 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/providers/http/nfqueue/nfqueue.go b/providers/http/nfqueue/nfqueue.go new file mode 100644 index 0000000000..4456170686 --- /dev/null +++ b/providers/http/nfqueue/nfqueue.go @@ -0,0 +1,196 @@ +// Package nfqueue implements a HTTP provider for solving the HTTP-01 challenge using nfqueue +// by captureing http challange pacet in fly and answering it by ourself +package nfqueue + +import ( + "bufio" + "bytes" + "context" + "fmt" + "log" + "net" + "net/http" + "os/exec" + "runtime" + "strings" + "time" + + gnfqueue "github.com/florianl/go-nfqueue" + "github.com/google/gopacket" + "github.com/google/gopacket/layers" +) + +// HTTPProvider implements HTTPProvider for `http-01` challenge. +type HTTPProvider struct { + port string + context context.Context + cancel context.CancelFunc +} + +// NewHttpDpiProvider returns a HTTPProvider instance with a configured port. +func NewHttpDpiProvider(port string) (*HTTPProvider, error) { + + c := &HTTPProvider{ + port: port, + } + + return c, nil +} + +// this craft acme challange response in HTTP level +func craftkeyauthresponse(keyAuth string) []byte { + var reply []byte + reply = fmt.Append(reply, "HTTP/1.1 200 OK\r\n") + reply = fmt.Append(reply, "Content-Type: text/plain\r\n") + reply = fmt.Append(reply, "server: go-acme-nfqueue\r\n") + reply = fmt.Appendf(reply, "Content-Length: %d\r\n", len(keyAuth)) + reply = fmt.Append(reply, "\r\n", keyAuth) + + return reply +} + +// craft packet +func craftReplyPacketBytes(keyAuth string, inputpacket gopacket.Packet) []byte { + outbuffer := gopacket.NewSerializeBuffer() + opt := gopacket.SerializeOptions{ + FixLengths: true, + ComputeChecksums: true, + } + inputTcp := inputpacket.Layer(layers.LayerTypeTCP).(*layers.TCP) + inputIPv4 := inputpacket.Layer(layers.LayerTypeIPv4).(*layers.IPv4) + + httplayer := gopacket.Payload(craftkeyauthresponse(keyAuth)) + tcplayer := &layers.TCP{ + // we reply back so reverse src and dst ports + SrcPort: inputTcp.DstPort, + DstPort: inputTcp.SrcPort, + Ack: inputTcp.Seq + uint32(len(inputTcp.Payload)), + Seq: inputTcp.Ack, + PSH: true, + ACK: true, + } + // log.Infof("dstp: %s, srcp %s", tcplayer.DstPort.String(), tcp) + //check network layer + // this is reply so we reverse sorce and dst ip + iplayer := &layers.IPv4{ + SrcIP: inputIPv4.DstIP, + DstIP: inputIPv4.SrcIP, + } + tcplayer.SetNetworkLayerForChecksum(iplayer) + gopacket.SerializeLayers(outbuffer, opt, tcplayer, httplayer) + + return outbuffer.Bytes() +} + +// sendPacket sends packet: TODO: call cleanup if errors out +func sendPacket(packet []byte, DstIP *net.IP) error { + var err error + con, err := net.Dial("ip:6", DstIP.String()) + if err != nil { + return err + } + _, err = con.Write(packet) + if err != nil { + return err + } + return nil +} + +// serve runs server by sniffing packets on firewall and inject response into it. +// iptables :// +func (w *HTTPProvider) serve(domain, token, keyAuth string) error { + //run nfqueue start + cmd := exec.Command("iptables", "-I", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555") + err := cmd.Run() + // ensure even if clean funtion failed to called + defer exec.Command("iptables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555").Run() + if err != nil { + return err + } + config := gnfqueue.Config{ + NfQueue: 8555, + MaxPacketLen: 0xFFFF, + MaxQueueLen: 0xFF, + Copymode: gnfqueue.NfQnlCopyPacket, + WriteTimeout: 15 * time.Millisecond, + } + nf, err := gnfqueue.Open(&config) + if err != nil { + return err + } + defer nf.Close() + + //handle Packet + handlepacket := func(a gnfqueue.Attribute) int { + id := *a.PacketID + opt := gopacket.DecodeOptions{ + NoCopy: true, + Lazy: false, + } + //assume ipv4 for now, will segfault + payload := gopacket.NewPacket(*a.Payload, layers.LayerTypeIPv4, opt) + ipL := payload.Layer(layers.LayerTypeIPv4) + srcip := ipL.(*layers.IPv4).SrcIP + if tcpLayer := payload.Layer(layers.LayerTypeTCP); tcpLayer != nil { + // Get actual TCP data from this layer + inputTcp, _ := tcpLayer.(*layers.TCP) + // this should be HTTP payload + httpPayload, err := http.ReadRequest(bufio.NewReader((bytes.NewReader(inputTcp.LayerPayload())))) + if err != nil { + nf.SetVerdict(id, gnfqueue.NfAccept) + return 0 + } + // check token in http + if strings.Contains(httpPayload.URL.Path, token) { + //we got the token!, block the packet to backend server. + nf.SetVerdict(id, gnfqueue.NfDrop) + //forge our new reply + replypacket := craftReplyPacketBytes(keyAuth, payload) + // Send the modified packet back to VA, ignore err as it won't crash + sendPacket(replypacket, &srcip) + // packet sent, end of function + return 0 + } else { + nf.SetVerdict(id, gnfqueue.NfAccept) + return 0 + } + + } else { + nf.SetVerdict(id, gnfqueue.NfAccept) + } + + return 0 + } + + // Register your function to listen on nflqueue queue + err = nf.Register(w.context, handlepacket) + if err != nil { + fmt.Println(err) + return nil + } + + // Block till the context expires + <-w.context.Done() + return nil +} + +func (w *HTTPProvider) Present(domain, token, keyAuth string) error { + // test if OS is linux, otherwise no point running this nfqueue is linux thing + if runtime.GOOS != "linux" { + log.Panicf("[%s] http-nfq provider isn't implimented non-linux", domain) + } + w.context, w.cancel = context.WithCancel(context.Background()) + go w.serve(domain, token, keyAuth) + return nil +} + +// CleanUp removes the firewall rule created for the challenge. +// solve should removed it already but just do be safe: +// iptables -D INPUT -p tcp --dport Port -j NFQUEUE --queue-num 8555 +func (w *HTTPProvider) CleanUp(domain, token, keyAuth string) error { + cmd := exec.Command("iptables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555") + cmd.Run() + // tell nfqueue to shut down + w.cancel() + return nil +} From 758045dd03a901ac170218cab854f98d8d05c822 Mon Sep 17 00:00:00 2001 From: Sascha Marcel Hacker Date: Mon, 20 Feb 2023 00:59:17 +0100 Subject: [PATCH 02/10] dynu: fix subdomain support (#1842) Co-authored-by: Fernandez Ludovic send RST to ACME server so it doens't retry use not deprecated version of func --- providers/http/nfqueue/nfqueue.go | 48 ++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/providers/http/nfqueue/nfqueue.go b/providers/http/nfqueue/nfqueue.go index 4456170686..fadabe25b7 100644 --- a/providers/http/nfqueue/nfqueue.go +++ b/providers/http/nfqueue/nfqueue.go @@ -50,7 +50,7 @@ func craftkeyauthresponse(keyAuth string) []byte { } // craft packet -func craftReplyPacketBytes(keyAuth string, inputpacket gopacket.Packet) []byte { +func craftReplyandSend(keyAuth string, inputpacket gopacket.Packet) error { outbuffer := gopacket.NewSerializeBuffer() opt := gopacket.SerializeOptions{ FixLengths: true, @@ -66,11 +66,12 @@ func craftReplyPacketBytes(keyAuth string, inputpacket gopacket.Packet) []byte { DstPort: inputTcp.SrcPort, Ack: inputTcp.Seq + uint32(len(inputTcp.Payload)), Seq: inputTcp.Ack, + Window: 1, PSH: true, ACK: true, } // log.Infof("dstp: %s, srcp %s", tcplayer.DstPort.String(), tcp) - //check network layer + // check network layer // this is reply so we reverse sorce and dst ip iplayer := &layers.IPv4{ SrcIP: inputIPv4.DstIP, @@ -78,8 +79,23 @@ func craftReplyPacketBytes(keyAuth string, inputpacket gopacket.Packet) []byte { } tcplayer.SetNetworkLayerForChecksum(iplayer) gopacket.SerializeLayers(outbuffer, opt, tcplayer, httplayer) + // send http reply + sendPacket(outbuffer.Bytes(), &iplayer.DstIP) - return outbuffer.Bytes() + // craft RST packet to server so connection can close by webserver + outbuffer.Clear() + tcplayer.RST = true + tcplayer.ACK = false + tcplayer.PSH = false + tcplayer.Seq = tcplayer.Seq + uint32(len(httplayer.Payload())) + + tcplayer.SetNetworkLayerForChecksum(iplayer) + gopacket.SerializeLayers(outbuffer, opt, tcplayer) + // rst to acme server so it knows it's done, + // our webserver will send some ack packet but it has misalign Seq so ignored by ACME server + sendPacket(outbuffer.Bytes(), &iplayer.DstIP) + + return nil } // sendPacket sends packet: TODO: call cleanup if errors out @@ -99,7 +115,7 @@ func sendPacket(packet []byte, DstIP *net.IP) error { // serve runs server by sniffing packets on firewall and inject response into it. // iptables :// func (w *HTTPProvider) serve(domain, token, keyAuth string) error { - //run nfqueue start + // run nfqueue start cmd := exec.Command("iptables", "-I", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555") err := cmd.Run() // ensure even if clean funtion failed to called @@ -120,17 +136,15 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { } defer nf.Close() - //handle Packet + // handle Packet handlepacket := func(a gnfqueue.Attribute) int { id := *a.PacketID opt := gopacket.DecodeOptions{ NoCopy: true, Lazy: false, } - //assume ipv4 for now, will segfault + // assume ipv4 for now, will segfault payload := gopacket.NewPacket(*a.Payload, layers.LayerTypeIPv4, opt) - ipL := payload.Layer(layers.LayerTypeIPv4) - srcip := ipL.(*layers.IPv4).SrcIP if tcpLayer := payload.Layer(layers.LayerTypeTCP); tcpLayer != nil { // Get actual TCP data from this layer inputTcp, _ := tcpLayer.(*layers.TCP) @@ -142,12 +156,13 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { } // check token in http if strings.Contains(httpPayload.URL.Path, token) { - //we got the token!, block the packet to backend server. + // we got the token!, block the packet to backend server. nf.SetVerdict(id, gnfqueue.NfDrop) - //forge our new reply - replypacket := craftReplyPacketBytes(keyAuth, payload) - // Send the modified packet back to VA, ignore err as it won't crash - sendPacket(replypacket, &srcip) + // forge our new reply + err := craftReplyandSend(keyAuth, payload) + if err != nil { + return 0 + } // packet sent, end of function return 0 } else { @@ -162,8 +177,13 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { return 0 } + ignoreerr := func(err error) int { + log.Print(err) + return 0 + } + // Register your function to listen on nflqueue queue - err = nf.Register(w.context, handlepacket) + err = nf.RegisterWithErrorFunc(w.context, handlepacket, ignoreerr) if err != nil { fmt.Println(err) return nil From b182087dc7fa46347c9c46799570621b36c10062 Mon Sep 17 00:00:00 2001 From: Seo Suchan Date: Wed, 22 Feb 2023 20:43:01 +0900 Subject: [PATCH 03/10] send rst to acme server to close tcp session --- providers/http/nfqueue/nfqueue.go | 48 ++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/providers/http/nfqueue/nfqueue.go b/providers/http/nfqueue/nfqueue.go index 4456170686..fadabe25b7 100644 --- a/providers/http/nfqueue/nfqueue.go +++ b/providers/http/nfqueue/nfqueue.go @@ -50,7 +50,7 @@ func craftkeyauthresponse(keyAuth string) []byte { } // craft packet -func craftReplyPacketBytes(keyAuth string, inputpacket gopacket.Packet) []byte { +func craftReplyandSend(keyAuth string, inputpacket gopacket.Packet) error { outbuffer := gopacket.NewSerializeBuffer() opt := gopacket.SerializeOptions{ FixLengths: true, @@ -66,11 +66,12 @@ func craftReplyPacketBytes(keyAuth string, inputpacket gopacket.Packet) []byte { DstPort: inputTcp.SrcPort, Ack: inputTcp.Seq + uint32(len(inputTcp.Payload)), Seq: inputTcp.Ack, + Window: 1, PSH: true, ACK: true, } // log.Infof("dstp: %s, srcp %s", tcplayer.DstPort.String(), tcp) - //check network layer + // check network layer // this is reply so we reverse sorce and dst ip iplayer := &layers.IPv4{ SrcIP: inputIPv4.DstIP, @@ -78,8 +79,23 @@ func craftReplyPacketBytes(keyAuth string, inputpacket gopacket.Packet) []byte { } tcplayer.SetNetworkLayerForChecksum(iplayer) gopacket.SerializeLayers(outbuffer, opt, tcplayer, httplayer) + // send http reply + sendPacket(outbuffer.Bytes(), &iplayer.DstIP) - return outbuffer.Bytes() + // craft RST packet to server so connection can close by webserver + outbuffer.Clear() + tcplayer.RST = true + tcplayer.ACK = false + tcplayer.PSH = false + tcplayer.Seq = tcplayer.Seq + uint32(len(httplayer.Payload())) + + tcplayer.SetNetworkLayerForChecksum(iplayer) + gopacket.SerializeLayers(outbuffer, opt, tcplayer) + // rst to acme server so it knows it's done, + // our webserver will send some ack packet but it has misalign Seq so ignored by ACME server + sendPacket(outbuffer.Bytes(), &iplayer.DstIP) + + return nil } // sendPacket sends packet: TODO: call cleanup if errors out @@ -99,7 +115,7 @@ func sendPacket(packet []byte, DstIP *net.IP) error { // serve runs server by sniffing packets on firewall and inject response into it. // iptables :// func (w *HTTPProvider) serve(domain, token, keyAuth string) error { - //run nfqueue start + // run nfqueue start cmd := exec.Command("iptables", "-I", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555") err := cmd.Run() // ensure even if clean funtion failed to called @@ -120,17 +136,15 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { } defer nf.Close() - //handle Packet + // handle Packet handlepacket := func(a gnfqueue.Attribute) int { id := *a.PacketID opt := gopacket.DecodeOptions{ NoCopy: true, Lazy: false, } - //assume ipv4 for now, will segfault + // assume ipv4 for now, will segfault payload := gopacket.NewPacket(*a.Payload, layers.LayerTypeIPv4, opt) - ipL := payload.Layer(layers.LayerTypeIPv4) - srcip := ipL.(*layers.IPv4).SrcIP if tcpLayer := payload.Layer(layers.LayerTypeTCP); tcpLayer != nil { // Get actual TCP data from this layer inputTcp, _ := tcpLayer.(*layers.TCP) @@ -142,12 +156,13 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { } // check token in http if strings.Contains(httpPayload.URL.Path, token) { - //we got the token!, block the packet to backend server. + // we got the token!, block the packet to backend server. nf.SetVerdict(id, gnfqueue.NfDrop) - //forge our new reply - replypacket := craftReplyPacketBytes(keyAuth, payload) - // Send the modified packet back to VA, ignore err as it won't crash - sendPacket(replypacket, &srcip) + // forge our new reply + err := craftReplyandSend(keyAuth, payload) + if err != nil { + return 0 + } // packet sent, end of function return 0 } else { @@ -162,8 +177,13 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { return 0 } + ignoreerr := func(err error) int { + log.Print(err) + return 0 + } + // Register your function to listen on nflqueue queue - err = nf.Register(w.context, handlepacket) + err = nf.RegisterWithErrorFunc(w.context, handlepacket, ignoreerr) if err != nil { fmt.Println(err) return nil From cddd0e63b5a1a16046d995e9dfacd0a1bc38eab6 Mon Sep 17 00:00:00 2001 From: Seo Suchan Date: Fri, 24 Feb 2023 16:41:11 +0900 Subject: [PATCH 04/10] support for IPv6 --- providers/http/nfqueue/nfqueue.go | 62 +++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/providers/http/nfqueue/nfqueue.go b/providers/http/nfqueue/nfqueue.go index 3e73149160..b03c335395 100644 --- a/providers/http/nfqueue/nfqueue.go +++ b/providers/http/nfqueue/nfqueue.go @@ -7,7 +7,6 @@ import ( "bytes" "context" "fmt" - "log" "net" "net/http" "os/exec" @@ -15,6 +14,8 @@ import ( "strings" "time" + "github.com/go-acme/lego/v4/log" + gnfqueue "github.com/florianl/go-nfqueue" "github.com/google/gopacket" "github.com/google/gopacket/layers" @@ -55,10 +56,10 @@ func craftkeyauthresponse(keyAuth string) []byte { } // craft packet -func craftReplyandSend(keyAuth string, inputpacket gopacket.Packet) error { +func craftReplyandSend(keyAuth string, inputpacket gopacket.Packet, dst net.IP) error { outbuffer := gopacket.NewSerializeBuffer() inputTcp := inputpacket.Layer(layers.LayerTypeTCP).(*layers.TCP) - inputIPv4 := inputpacket.Layer(layers.LayerTypeIPv4).(*layers.IPv4) + inputIPL := inputpacket.NetworkLayer() httplayer := gopacket.Payload(craftkeyauthresponse(keyAuth)) tcplayer := &layers.TCP{ @@ -74,14 +75,11 @@ func craftReplyandSend(keyAuth string, inputpacket gopacket.Packet) error { // log.Infof("dstp: %s, srcp %s", tcplayer.DstPort.String(), tcp) // check network layer // this is reply so we reverse sorce and dst ip - iplayer := &layers.IPv4{ - SrcIP: inputIPv4.DstIP, - DstIP: inputIPv4.SrcIP, - } - tcplayer.SetNetworkLayerForChecksum(iplayer) + + tcplayer.SetNetworkLayerForChecksum(inputIPL) gopacket.SerializeLayers(outbuffer, sopt, tcplayer, httplayer) // send http reply - sendPacket(outbuffer.Bytes(), &iplayer.DstIP) + sendPacket(outbuffer.Bytes(), &dst) // craft RST packet to server so connection can close by webserver outbuffer.Clear() @@ -90,18 +88,18 @@ func craftReplyandSend(keyAuth string, inputpacket gopacket.Packet) error { tcplayer.PSH = false tcplayer.Seq = tcplayer.Seq + uint32(len(httplayer.Payload())) - tcplayer.SetNetworkLayerForChecksum(iplayer) + tcplayer.SetNetworkLayerForChecksum(inputIPL) gopacket.SerializeLayers(outbuffer, sopt, tcplayer) // rst to acme server so it knows it's done, // our webserver will send some ack packet but it has misalign Seq so ignored by ACME server - sendPacket(outbuffer.Bytes(), &iplayer.DstIP) + sendPacket(outbuffer.Bytes(), &dst) return nil } -func craftRSTbyte4(inpkt gopacket.Packet) []byte { +func craftRSTbyte(inpkt gopacket.Packet) []byte { tcpl := inpkt.Layer(layers.LayerTypeTCP).(*layers.TCP) - ipl := inpkt.Layer(layers.LayerTypeIPv4).(*layers.IPv4) + ipl := inpkt.LayerClass(layers.LayerClassIPNetwork).(gopacket.SerializableLayer) buf := gopacket.NewSerializeBuffer() tcpl.RST = true gopacket.SerializeLayers(buf, sopt, ipl, tcpl) @@ -127,9 +125,14 @@ func sendPacket(packet []byte, DstIP *net.IP) error { func (w *HTTPProvider) serve(domain, token, keyAuth string) error { // run nfqueue start cmd := exec.Command("iptables", "-I", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555") + defer exec.Command("iptables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555").Run() err := cmd.Run() + if err != nil { + return err + } + err = exec.Command("ip6tables", "-I", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555").Run() // ensure even if clean funtion failed to called - defer exec.Command("iptables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555").Run() + defer exec.Command("ip6tables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555").Run() if err != nil { return err } @@ -138,6 +141,7 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { MaxPacketLen: 0xFFFF, MaxQueueLen: 0xFF, Copymode: gnfqueue.NfQnlCopyPacket, + Flags: gnfqueue.NfQaCfgFlagFailOpen, WriteTimeout: 15 * time.Millisecond, } nf, err := gnfqueue.Open(&config) @@ -154,10 +158,27 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { NoCopy: true, Lazy: false, } - payload := gopacket.NewPacket(*a.Payload, layers.LayerTypeIPv4, dopt) - tcpLayer := payload.Layer(layers.LayerTypeTCP) + var ipLType gopacket.LayerType + if *a.HwProtocol == 0x0800 { + //ipv4 + ipLType = layers.LayerTypeIPv4 + } else if *a.HwProtocol == 0x86DD { + ipLType = layers.LayerTypeIPv6 + } else { + nf.SetVerdict(id, gnfqueue.NfAccept) + return 0 + } + payload := gopacket.NewPacket(*a.Payload, ipLType, dopt) + // iplayer := payload.LayerClass(layers.LayerClassIPNetwork) // Get actual TCP data from this layer + tcpLayer := payload.Layer(layers.LayerTypeTCP) + if tcpLayer == nil { + nf.SetVerdict(id, gnfqueue.NfAccept) + return 0 + } inputTcp := tcpLayer.(*layers.TCP) + // get destination IP here, this is sent from other side, so src is other side + otherend := net.IP(payload.NetworkLayer().NetworkFlow().Src().Raw()) // this should be HTTP payload httpPayload, err := http.ReadRequest(bufio.NewReader((bytes.NewReader(inputTcp.LayerPayload())))) if err != nil { @@ -168,7 +189,8 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { if strings.Contains(httpPayload.URL.Path, token) { // we got the token! // forge our new reply - err := craftReplyandSend(keyAuth, payload) + log.Infof("[%s] Injecting key authentication", domain) + err := craftReplyandSend(keyAuth, payload, otherend) if err != nil { return 0 } @@ -177,7 +199,7 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { fmt.Print("modpacket err", err) } - rstpk := craftRSTbyte4(payload) + rstpk := craftRSTbyte(payload) err = nf.SetVerdictModPacket(id, gnfqueue.NfAccept, rstpk) if err != nil { fmt.Print("modpacket err", err) @@ -210,7 +232,7 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { func (w *HTTPProvider) Present(domain, token, keyAuth string) error { // test if OS is linux, otherwise no point running this nfqueue is linux thing if runtime.GOOS != "linux" { - log.Panicf("[%s] http-nfq provider isn't implimented non-linux", domain) + log.Fatalf("[%s] http-nfq provider isn't implimented non-linux", domain) } w.context, w.cancel = context.WithCancel(context.Background()) go w.serve(domain, token, keyAuth) @@ -223,6 +245,8 @@ func (w *HTTPProvider) Present(domain, token, keyAuth string) error { func (w *HTTPProvider) CleanUp(domain, token, keyAuth string) error { cmd := exec.Command("iptables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555") cmd.Run() + cmd = exec.Command("ip6tables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555") + cmd.Run() // tell nfqueue to shut down w.cancel() return nil From 3a70fb241dac47001a20870f263397dc5d86e8d0 Mon Sep 17 00:00:00 2001 From: Seo Suchan Date: Fri, 24 Feb 2023 21:14:24 +0900 Subject: [PATCH 05/10] make firewall nonblocking even if lego crashed --- providers/http/nfqueue/nfqueue.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/providers/http/nfqueue/nfqueue.go b/providers/http/nfqueue/nfqueue.go index b03c335395..2d4520ddbe 100644 --- a/providers/http/nfqueue/nfqueue.go +++ b/providers/http/nfqueue/nfqueue.go @@ -11,7 +11,6 @@ import ( "net/http" "os/exec" "runtime" - "strings" "time" "github.com/go-acme/lego/v4/log" @@ -124,15 +123,15 @@ func sendPacket(packet []byte, DstIP *net.IP) error { // iptables :// func (w *HTTPProvider) serve(domain, token, keyAuth string) error { // run nfqueue start - cmd := exec.Command("iptables", "-I", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555") - defer exec.Command("iptables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555").Run() + cmd := exec.Command("iptables", "-I", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass") + defer exec.Command("iptables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass").Run() err := cmd.Run() if err != nil { return err } - err = exec.Command("ip6tables", "-I", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555").Run() + err = exec.Command("ip6tables", "-I", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass").Run() // ensure even if clean funtion failed to called - defer exec.Command("ip6tables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555").Run() + defer exec.Command("ip6tables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass").Run() if err != nil { return err } @@ -186,7 +185,8 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { return 0 } // check token in http - if strings.Contains(httpPayload.URL.Path, token) { + chalPath := fmt.Sprintf("/.well-known/acme-challenge/%s", token) + if httpPayload.URL.Path == chalPath { // we got the token! // forge our new reply log.Infof("[%s] Injecting key authentication", domain) @@ -243,9 +243,9 @@ func (w *HTTPProvider) Present(domain, token, keyAuth string) error { // solve should removed it already but just do be safe: // iptables -D INPUT -p tcp --dport Port -j NFQUEUE --queue-num 8555 func (w *HTTPProvider) CleanUp(domain, token, keyAuth string) error { - cmd := exec.Command("iptables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555") + cmd := exec.Command("iptables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass") cmd.Run() - cmd = exec.Command("ip6tables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555") + cmd = exec.Command("ip6tables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass") cmd.Run() // tell nfqueue to shut down w.cancel() From 92099e08e4889c93d40f29933632684ecac0091e Mon Sep 17 00:00:00 2001 From: Seo Suchan Date: Sat, 25 Feb 2023 20:22:58 +0900 Subject: [PATCH 06/10] Do TCP graceful shutdown instead of rst --- providers/http/nfqueue/nfqueue.go | 33 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/providers/http/nfqueue/nfqueue.go b/providers/http/nfqueue/nfqueue.go index 2d4520ddbe..21ce433e86 100644 --- a/providers/http/nfqueue/nfqueue.go +++ b/providers/http/nfqueue/nfqueue.go @@ -70,27 +70,27 @@ func craftReplyandSend(keyAuth string, inputpacket gopacket.Packet, dst net.IP) Window: 1, PSH: true, ACK: true, + // we want to finish TCP after this packet so set fin + FIN: true, } - // log.Infof("dstp: %s, srcp %s", tcplayer.DstPort.String(), tcp) - // check network layer - // this is reply so we reverse sorce and dst ip - + // answer is same with same protocal, so we use input's layer tcplayer.SetNetworkLayerForChecksum(inputIPL) gopacket.SerializeLayers(outbuffer, sopt, tcplayer, httplayer) // send http reply sendPacket(outbuffer.Bytes(), &dst) - // craft RST packet to server so connection can close by webserver + // need to ACK the server FIN so acme server can close connection outbuffer.Clear() - tcplayer.RST = true - tcplayer.ACK = false + tcplayer.ACK = true tcplayer.PSH = false - tcplayer.Seq = tcplayer.Seq + uint32(len(httplayer.Payload())) + tcplayer.Seq = tcplayer.Seq + uint32(len(httplayer.Payload())) + 1 + tcplayer.Ack = inputTcp.Seq + uint32(len(inputTcp.Payload)) + 1 tcplayer.SetNetworkLayerForChecksum(inputIPL) gopacket.SerializeLayers(outbuffer, sopt, tcplayer) - // rst to acme server so it knows it's done, - // our webserver will send some ack packet but it has misalign Seq so ignored by ACME server + // Fin+ACK to acme server so it knows it's done, + // sleep some time here so acme server sent it's FIN+ACK here + time.Sleep(time.Millisecond * 10) sendPacket(outbuffer.Bytes(), &dst) return nil @@ -98,9 +98,11 @@ func craftReplyandSend(keyAuth string, inputpacket gopacket.Packet, dst net.IP) func craftRSTbyte(inpkt gopacket.Packet) []byte { tcpl := inpkt.Layer(layers.LayerTypeTCP).(*layers.TCP) + tcpl.SetNetworkLayerForChecksum(inpkt.NetworkLayer()) ipl := inpkt.LayerClass(layers.LayerClassIPNetwork).(gopacket.SerializableLayer) buf := gopacket.NewSerializeBuffer() tcpl.RST = true + tcpl.ACK = true gopacket.SerializeLayers(buf, sopt, ipl, tcpl) return buf.Bytes() } @@ -167,17 +169,17 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { nf.SetVerdict(id, gnfqueue.NfAccept) return 0 } - payload := gopacket.NewPacket(*a.Payload, ipLType, dopt) + packetin := gopacket.NewPacket(*a.Payload, ipLType, dopt) // iplayer := payload.LayerClass(layers.LayerClassIPNetwork) // Get actual TCP data from this layer - tcpLayer := payload.Layer(layers.LayerTypeTCP) + tcpLayer := packetin.Layer(layers.LayerTypeTCP) if tcpLayer == nil { nf.SetVerdict(id, gnfqueue.NfAccept) return 0 } inputTcp := tcpLayer.(*layers.TCP) // get destination IP here, this is sent from other side, so src is other side - otherend := net.IP(payload.NetworkLayer().NetworkFlow().Src().Raw()) + otherend := net.IP(packetin.NetworkLayer().NetworkFlow().Src().Raw()) // this should be HTTP payload httpPayload, err := http.ReadRequest(bufio.NewReader((bytes.NewReader(inputTcp.LayerPayload())))) if err != nil { @@ -190,7 +192,7 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { // we got the token! // forge our new reply log.Infof("[%s] Injecting key authentication", domain) - err := craftReplyandSend(keyAuth, payload, otherend) + err := craftReplyandSend(keyAuth, packetin, otherend) if err != nil { return 0 } @@ -198,8 +200,7 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { if err != nil { fmt.Print("modpacket err", err) } - - rstpk := craftRSTbyte(payload) + rstpk := craftRSTbyte(packetin) err = nf.SetVerdictModPacket(id, gnfqueue.NfAccept, rstpk) if err != nil { fmt.Print("modpacket err", err) From 2cd41354206cc082dfafb9d39a33982e69649bc5 Mon Sep 17 00:00:00 2001 From: Seo Suchan Date: Sun, 26 Feb 2023 17:50:55 +0900 Subject: [PATCH 07/10] [std-nfq]assert a server is listen on port --- providers/http/nfqueue/nfqueue.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/providers/http/nfqueue/nfqueue.go b/providers/http/nfqueue/nfqueue.go index 21ce433e86..afb69ac567 100644 --- a/providers/http/nfqueue/nfqueue.go +++ b/providers/http/nfqueue/nfqueue.go @@ -233,7 +233,14 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { func (w *HTTPProvider) Present(domain, token, keyAuth string) error { // test if OS is linux, otherwise no point running this nfqueue is linux thing if runtime.GOOS != "linux" { - log.Fatalf("[%s] http-nfq provider isn't implimented non-linux", domain) + return fmt.Errorf("[%s] http-nfq provider isn't implimented non-linux", domain) + } + // test if there is a webserver on port requested + con, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%s", w.port), time.Second) + if err != nil { + return fmt.Errorf("[%s] http-nfq needs a webserver watching on requested, port %s", domain, w.port) + } else { + con.Close() } w.context, w.cancel = context.WithCancel(context.Background()) go w.serve(domain, token, keyAuth) From 2ffee23490ba1654960345f4bad356374be6528d Mon Sep 17 00:00:00 2001 From: Seo Suchan Date: Mon, 27 Feb 2023 14:57:58 +0900 Subject: [PATCH 08/10] make firewall rule a function --- providers/http/nfqueue/nfqueue.go | 42 +++++++++++++++----------- providers/http/nfqueue/nfqueue_test.go | 19 ++++++++++++ 2 files changed, 44 insertions(+), 17 deletions(-) create mode 100644 providers/http/nfqueue/nfqueue_test.go diff --git a/providers/http/nfqueue/nfqueue.go b/providers/http/nfqueue/nfqueue.go index afb69ac567..3adf0749a7 100644 --- a/providers/http/nfqueue/nfqueue.go +++ b/providers/http/nfqueue/nfqueue.go @@ -54,6 +54,25 @@ func craftkeyauthresponse(keyAuth string) []byte { return reply } +func setFirewallRule(on bool, port string) error { + var onoff string + if on { + onoff = "-I" + } else { + onoff = "-D" + } + + out, err := exec.Command("sudo", "iptables", onoff, "INPUT", "-p", "tcp", "--dport", port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass").CombinedOutput() + if err != nil { + return fmt.Errorf("%s", out) + } + err = exec.Command("sudo", "ip6tables", onoff, "INPUT", "-p", "tcp", "--dport", port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass").Run() + if err != nil { + return fmt.Errorf("%s", out) + } + return nil +} + // craft packet func craftReplyandSend(keyAuth string, inputpacket gopacket.Packet, dst net.IP) error { outbuffer := gopacket.NewSerializeBuffer() @@ -125,18 +144,6 @@ func sendPacket(packet []byte, DstIP *net.IP) error { // iptables :// func (w *HTTPProvider) serve(domain, token, keyAuth string) error { // run nfqueue start - cmd := exec.Command("iptables", "-I", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass") - defer exec.Command("iptables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass").Run() - err := cmd.Run() - if err != nil { - return err - } - err = exec.Command("ip6tables", "-I", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass").Run() - // ensure even if clean funtion failed to called - defer exec.Command("ip6tables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass").Run() - if err != nil { - return err - } config := gnfqueue.Config{ NfQueue: 8555, MaxPacketLen: 0xFFFF, @@ -233,7 +240,7 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { func (w *HTTPProvider) Present(domain, token, keyAuth string) error { // test if OS is linux, otherwise no point running this nfqueue is linux thing if runtime.GOOS != "linux" { - return fmt.Errorf("[%s] http-nfq provider isn't implimented non-linux", domain) + return fmt.Errorf("[%s] http-nfq provider is only for linux", domain) } // test if there is a webserver on port requested con, err := net.DialTimeout("tcp", fmt.Sprintf("localhost:%s", w.port), time.Second) @@ -242,6 +249,10 @@ func (w *HTTPProvider) Present(domain, token, keyAuth string) error { } else { con.Close() } + err = setFirewallRule(true, w.port) + if err != nil { + return err + } w.context, w.cancel = context.WithCancel(context.Background()) go w.serve(domain, token, keyAuth) return nil @@ -251,10 +262,7 @@ func (w *HTTPProvider) Present(domain, token, keyAuth string) error { // solve should removed it already but just do be safe: // iptables -D INPUT -p tcp --dport Port -j NFQUEUE --queue-num 8555 func (w *HTTPProvider) CleanUp(domain, token, keyAuth string) error { - cmd := exec.Command("iptables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass") - cmd.Run() - cmd = exec.Command("ip6tables", "-D", "INPUT", "-p", "tcp", "--dport", w.port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass") - cmd.Run() + setFirewallRule(false, w.port) // tell nfqueue to shut down w.cancel() return nil diff --git a/providers/http/nfqueue/nfqueue_test.go b/providers/http/nfqueue/nfqueue_test.go new file mode 100644 index 0000000000..70a9b86e3e --- /dev/null +++ b/providers/http/nfqueue/nfqueue_test.go @@ -0,0 +1,19 @@ +package nfqueue + +import ( + "runtime" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNonLinux(t *testing.T) { + // this test doesn't apply for linux + if runtime.GOOS == "linux" { + return + } + serv, _ := NewHttpDpiProvider("3331") + err := serv.Present("exemple.org", "somerandomstring", "otherrandomstring") + // just test if error mentions linux here + assert.Contains(t, err.Error(), "linux") +} From f86d02f93f718b29155ade4059a8c5f445251e2f Mon Sep 17 00:00:00 2001 From: Seo Suchan Date: Wed, 1 Mar 2023 16:45:55 +0900 Subject: [PATCH 09/10] add test(need root) --- providers/http/nfqueue/nfqueue_linux_test.go | 114 +++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 providers/http/nfqueue/nfqueue_linux_test.go diff --git a/providers/http/nfqueue/nfqueue_linux_test.go b/providers/http/nfqueue/nfqueue_linux_test.go new file mode 100644 index 0000000000..c971ee08cc --- /dev/null +++ b/providers/http/nfqueue/nfqueue_linux_test.go @@ -0,0 +1,114 @@ +//go:build root +// +build root + +// this tests need to run as root to function + +package nfqueue + +import ( + "crypto/rand" + "crypto/rsa" + "io" + "net/http" + "testing" + "time" + + "github.com/go-acme/lego/v4/acme" + "github.com/go-acme/lego/v4/acme/api" + "github.com/go-acme/lego/v4/challenge" + "github.com/go-acme/lego/v4/challenge/http01" + "github.com/go-acme/lego/v4/platform/tester" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var testpayload = []byte("this is the server behind") + +func simpleHttp(port string) { + sv := http.Server{ + Addr: ":" + port, + Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.Write(testpayload) + }), + } + + sv.ListenAndServe() +} + +// this test our firewall rule doesn't hinder other webeserver traffic +func TestNotHinder(t *testing.T) { + port := "31234" + prv, _ := NewHttpDpiProvider("31234") + go simpleHttp(port) + prv.Present("labmdawork", "sampletoken", "keyauth") + defer prv.CleanUp("labmdawork", "sampletoken", "keyauth") + resp, err := http.Get("http://127.0.0.1:31234/hello") + if err != nil { + panic(err) + } + respBody, err := io.ReadAll(resp.Body) + assert.Nil(t, err) + assert.Equal(t, testpayload, respBody) + +} + +func TestFirewallSet(t *testing.T) { + err := setFirewallRule(true, "12345") + assert.Nil(t, err) + defer setFirewallRule(false, "12345") +} + +func TestChallengeinner(t *testing.T) { + _, apiURL := tester.SetupFakeAPI(t) + + providerServer, _ := NewHttpDpiProvider("23457") + go simpleHttp("23457") + time.Sleep(50 * time.Microsecond) + _, err := http.Get("http://127.0.0.1:23457/hello") + assert.Nil(t, err) + validate := func(_ *api.Core, _ string, chlng acme.Challenge) error { + uri := "http://localhost" + ":23457" + http01.ChallengePath(chlng.Token) + + resp, err := http.DefaultClient.Get(uri) + if err != nil { + return err + } + defer resp.Body.Close() + + if want := "text/plain"; resp.Header.Get("Content-Type") != want { + t.Errorf("Get(%q) Content-Type: got %q, want %q", uri, resp.Header.Get("Content-Type"), want) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + bodyStr := string(body) + + if bodyStr != chlng.KeyAuthorization { + t.Errorf("Get(%q) Body: got %q, want %q", uri, bodyStr, chlng.KeyAuthorization) + } + + return nil + } + + privateKey, err := rsa.GenerateKey(rand.Reader, 512) + require.NoError(t, err, "Could not generate test key") + + core, err := api.New(http.DefaultClient, "lego-test", apiURL+"/dir", "", privateKey) + require.NoError(t, err) + + solver := http01.NewChallenge(core, validate, providerServer) + + authz := acme.Authorization{ + Identifier: acme.Identifier{ + Value: "localhost:23457", + }, + Challenges: []acme.Challenge{ + {Type: challenge.HTTP01.String(), Token: "http1"}, + }, + } + + err = solver.Solve(authz) + require.NoError(t, err) +} From b2681c32ea629bed319328fa4a24b47a79021a82 Mon Sep 17 00:00:00 2001 From: Seo Suchan Date: Wed, 1 Mar 2023 16:46:09 +0900 Subject: [PATCH 10/10] make handlepacket own function --- providers/http/nfqueue/nfqueue.go | 163 +++++++++++++++++------------- 1 file changed, 91 insertions(+), 72 deletions(-) diff --git a/providers/http/nfqueue/nfqueue.go b/providers/http/nfqueue/nfqueue.go index 3adf0749a7..336b377669 100644 --- a/providers/http/nfqueue/nfqueue.go +++ b/providers/http/nfqueue/nfqueue.go @@ -1,5 +1,6 @@ // Package nfqueue implements a HTTP provider for solving the HTTP-01 challenge using nfqueue // by captureing http challange pacet in fly and answering it by ourself +// This solver needs a TCP server attached on request port, and need root or CAP_NET_ADMIN package nfqueue import ( @@ -27,6 +28,14 @@ type HTTPProvider struct { cancel context.CancelFunc } +// a struct holds thing that packet handler should know about +type chalnfq struct { + domain string + token string + keyAuth string + queue *gnfqueue.Nfqueue +} + var sopt = gopacket.SerializeOptions{ FixLengths: true, ComputeChecksums: true, @@ -42,7 +51,7 @@ func NewHttpDpiProvider(port string) (*HTTPProvider, error) { return c, nil } -// this craft acme challange response in HTTP level +// craftkeyauthresponse carft acme challange response in HTTP level func craftkeyauthresponse(keyAuth string) []byte { var reply []byte reply = fmt.Append(reply, "HTTP/1.1 200 OK\r\n") @@ -54,19 +63,22 @@ func craftkeyauthresponse(keyAuth string) []byte { return reply } +// setFirewallRule set rule in firewall INPUT chain so we can sniff on +// with --queue-bypass option even if this crash without clean webserver will listen +// iptables {on} INPUT -p tcp --dport {Port} -j NFQUEUE --queue-num 8555 --queue-bypass func setFirewallRule(on bool, port string) error { + // google's nft api is unstable, so we run command as-is var onoff string if on { onoff = "-I" } else { onoff = "-D" } - - out, err := exec.Command("sudo", "iptables", onoff, "INPUT", "-p", "tcp", "--dport", port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass").CombinedOutput() + out, err := exec.Command("iptables", onoff, "INPUT", "-p", "tcp", "--dport", port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass").CombinedOutput() if err != nil { return fmt.Errorf("%s", out) } - err = exec.Command("sudo", "ip6tables", onoff, "INPUT", "-p", "tcp", "--dport", port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass").Run() + err = exec.Command("ip6tables", onoff, "INPUT", "-p", "tcp", "--dport", port, "-j", "NFQUEUE", "--queue-num", "8555", "--queue-bypass").Run() if err != nil { return fmt.Errorf("%s", out) } @@ -98,7 +110,7 @@ func craftReplyandSend(keyAuth string, inputpacket gopacket.Packet, dst net.IP) // send http reply sendPacket(outbuffer.Bytes(), &dst) - // need to ACK the server FIN so acme server can close connection + // need to ACK for the server FIN so acme server can close connection outbuffer.Clear() tcplayer.ACK = true tcplayer.PSH = false @@ -107,8 +119,8 @@ func craftReplyandSend(keyAuth string, inputpacket gopacket.Packet, dst net.IP) tcplayer.SetNetworkLayerForChecksum(inputIPL) gopacket.SerializeLayers(outbuffer, sopt, tcplayer) - // Fin+ACK to acme server so it knows it's done, - // sleep some time here so acme server sent it's FIN+ACK here + // sleep some time here so acme server sent its FIN+ACK when this arrives + // alternatives are: 1. send rst here instead 2. actually trace connection time.Sleep(time.Millisecond * 10) sendPacket(outbuffer.Bytes(), &dst) @@ -140,8 +152,66 @@ func sendPacket(packet []byte, DstIP *net.IP) error { return nil } +// handlePacket handles packet input +func (q chalnfq) handlePacket(qupkt gnfqueue.Attribute) int { + id := *qupkt.PacketID + dopt := gopacket.DecodeOptions{ + NoCopy: true, + Lazy: false, + } + var ipLType gopacket.LayerType + // Hwprotocol here is ethernet frame protocol header + if *qupkt.HwProtocol == 0x0800 { + //ipv4 + ipLType = layers.LayerTypeIPv4 + } else if *qupkt.HwProtocol == 0x86DD { + ipLType = layers.LayerTypeIPv6 + } else { + q.queue.SetVerdict(id, gnfqueue.NfAccept) + return 0 + } + packetin := gopacket.NewPacket(*qupkt.Payload, ipLType, dopt) + // Get actual TCP data from this layer + tcpLayer := packetin.Layer(layers.LayerTypeTCP) + if tcpLayer == nil { + q.queue.SetVerdict(id, gnfqueue.NfAccept) + return 0 + } + inputTcp := tcpLayer.(*layers.TCP) + // get destination IP here, this is sent from other side, so src is other side + otherend := net.IP(packetin.NetworkLayer().NetworkFlow().Src().Raw()) + // this should be HTTP payload as this is webserver + httpPayload, err := http.ReadRequest(bufio.NewReader((bytes.NewReader(inputTcp.LayerPayload())))) + if err != nil { + q.queue.SetVerdict(id, gnfqueue.NfAccept) + return 0 + } + // check this request ask for token + chalPath := fmt.Sprintf("/.well-known/acme-challenge/%s", q.token) + if httpPayload.URL.Path == chalPath { + // we got the token! + // forge our new reply + log.Infof("[%s] Injecting key authentication", q.domain) + err := craftReplyandSend(q.keyAuth, packetin, otherend) + if err != nil { + return 0 + } + // mark incomming packet as RST so backend server ignore and close session + rstpk := craftRSTbyte(packetin) + err = q.queue.SetVerdictModPacket(id, gnfqueue.NfAccept, rstpk) + if err != nil { + fmt.Print("modpacket err", err) + } + // packet sent, end of function + return 0 + } else { + q.queue.SetVerdict(id, gnfqueue.NfAccept) + return 0 + } + +} + // serve runs server by sniffing packets on firewall and inject response into it. -// iptables :// func (w *HTTPProvider) serve(domain, token, keyAuth string) error { // run nfqueue start config := gnfqueue.Config{ @@ -158,75 +228,21 @@ func (w *HTTPProvider) serve(domain, token, keyAuth string) error { } defer nf.Close() - // handle Packet - handlepacket := func(a gnfqueue.Attribute) int { - id := *a.PacketID - // assume ipv4 for now, will segfault - dopt := gopacket.DecodeOptions{ - NoCopy: true, - Lazy: false, - } - var ipLType gopacket.LayerType - if *a.HwProtocol == 0x0800 { - //ipv4 - ipLType = layers.LayerTypeIPv4 - } else if *a.HwProtocol == 0x86DD { - ipLType = layers.LayerTypeIPv6 - } else { - nf.SetVerdict(id, gnfqueue.NfAccept) - return 0 - } - packetin := gopacket.NewPacket(*a.Payload, ipLType, dopt) - // iplayer := payload.LayerClass(layers.LayerClassIPNetwork) - // Get actual TCP data from this layer - tcpLayer := packetin.Layer(layers.LayerTypeTCP) - if tcpLayer == nil { - nf.SetVerdict(id, gnfqueue.NfAccept) - return 0 - } - inputTcp := tcpLayer.(*layers.TCP) - // get destination IP here, this is sent from other side, so src is other side - otherend := net.IP(packetin.NetworkLayer().NetworkFlow().Src().Raw()) - // this should be HTTP payload - httpPayload, err := http.ReadRequest(bufio.NewReader((bytes.NewReader(inputTcp.LayerPayload())))) - if err != nil { - nf.SetVerdict(id, gnfqueue.NfAccept) - return 0 - } - // check token in http - chalPath := fmt.Sprintf("/.well-known/acme-challenge/%s", token) - if httpPayload.URL.Path == chalPath { - // we got the token! - // forge our new reply - log.Infof("[%s] Injecting key authentication", domain) - err := craftReplyandSend(keyAuth, packetin, otherend) - if err != nil { - return 0 - } - // mark incomming packet as RST so backend server ignore and close session - if err != nil { - fmt.Print("modpacket err", err) - } - rstpk := craftRSTbyte(packetin) - err = nf.SetVerdictModPacket(id, gnfqueue.NfAccept, rstpk) - if err != nil { - fmt.Print("modpacket err", err) - } - // packet sent, end of function - return 0 - } else { - nf.SetVerdict(id, gnfqueue.NfAccept) - return 0 - } + h := chalnfq{ + token: token, + domain: domain, + keyAuth: keyAuth, + queue: nf, } + // error here would mean we couldn't capture packet, notthing to act about ignoreerr := func(err error) int { log.Print(err) return 0 } - // Register your function to listen on nflqueue queue - err = nf.RegisterWithErrorFunc(w.context, handlepacket, ignoreerr) + // Register function to listen on nflqueue queue + err = nf.RegisterWithErrorFunc(w.context, h.handlePacket, ignoreerr) if err != nil { fmt.Println(err) return nil @@ -249,9 +265,13 @@ func (w *HTTPProvider) Present(domain, token, keyAuth string) error { } else { con.Close() } + // if there is residuel firewall rule from old run remove it, ignore error + setFirewallRule(false, w.port) + + // try set actuall firewall rule needed err = setFirewallRule(true, w.port) if err != nil { - return err + return fmt.Errorf("[nfqueue] fail to set firewal rule, error : %s", err.Error()) } w.context, w.cancel = context.WithCancel(context.Background()) go w.serve(domain, token, keyAuth) @@ -260,7 +280,6 @@ func (w *HTTPProvider) Present(domain, token, keyAuth string) error { // CleanUp removes the firewall rule created for the challenge. // solve should removed it already but just do be safe: -// iptables -D INPUT -p tcp --dport Port -j NFQUEUE --queue-num 8555 func (w *HTTPProvider) CleanUp(domain, token, keyAuth string) error { setFirewallRule(false, w.port) // tell nfqueue to shut down