From 6611b7a4e1780c2061631dd435ed75de29e3f73d Mon Sep 17 00:00:00 2001 From: Gregor Pogacnik <1640719+fiksn@users.noreply.github.com> Date: Fri, 14 Feb 2020 00:15:21 +0100 Subject: [PATCH 1/4] Add experimental TLS support (MITM as a service) --- README.md | 23 ++++++++++++ proxy.go | 91 +++++++++++++++++++++++++++++++++++++-------- proxy_collection.go | 2 + 3 files changed, 101 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index fc7843f3..208c5230 100644 --- a/README.md +++ b/README.md @@ -309,6 +309,29 @@ An example `config/toxiproxy.json`: ] ``` +An example `config/toxiproxy.json` with the experimental TLS feature: + +```json +[ + { + "name": "plain", + "listen": "[::]:1080", + "upstream": "www.arnes.si:80", + "enabled": true + }, + { + "name": "ssl", + "listen": "[::]:1443", + "upstream": "www.arnes.si:443", + "enabled": true, + "tls": { + "cert": "./cert.crt", + "key": "./cert.key" + } + } +] +``` + Use ports outside the ephemeral port range to avoid random port conflicts. It's `32,768` to `61,000` on Linux by default, see `/proc/sys/net/ipv4/ip_local_port_range`. diff --git a/proxy.go b/proxy.go index 1d165053..63961889 100644 --- a/proxy.go +++ b/proxy.go @@ -1,6 +1,8 @@ package toxiproxy import ( + "crypto/rand" + "crypto/tls" "errors" "sync" @@ -20,10 +22,11 @@ import ( type Proxy struct { sync.Mutex - Name string `json:"name"` - Listen string `json:"listen"` - Upstream string `json:"upstream"` - Enabled bool `json:"enabled"` + Name string `json:"name"` + Listen string `json:"listen"` + Upstream string `json:"upstream"` + Enabled bool `json:"enabled"` + TLS *TlsData `json:"tls,omitempty"` started chan error @@ -32,6 +35,11 @@ type Proxy struct { Toxics *ToxicCollection `json:"-"` } +type TlsData struct { + Cert string `json:"cert"` + Key string `json:"key"` +} + type ConnectionList struct { list map[string]net.Conn lock sync.Mutex @@ -92,7 +100,38 @@ func (proxy *Proxy) Stop() { // server runs the Proxy server, accepting new clients and creating Links to // connect them to upstreams. func (proxy *Proxy) server() { - ln, err := net.Listen("tcp", proxy.Listen) + var ( + ln net.Listener + err error + upstream net.Conn + ) + + if proxy.TLS != nil { + logrus.WithFields(logrus.Fields{ + "proxy": proxy.Name, + "cert": proxy.TLS.Cert, + "key": proxy.TLS.Key, + }).Info("TLS certificates were specified") + } else { + logrus.WithFields(logrus.Fields{ + "proxy": proxy.Name, + }).Info("TLS certificates were NOT specified") + } + + if proxy.TLS != nil { + cert, err := tls.LoadX509KeyPair(proxy.TLS.Cert, proxy.TLS.Key) + if err != nil { + proxy.started <- err + return + } + + config := tls.Config{Certificates: []tls.Certificate{cert}} + config.Rand = rand.Reader + ln, err = tls.Listen("tcp", proxy.Listen, &config) + } else { + ln, err = net.Listen("tcp", proxy.Listen) + } + if err != nil { proxy.started <- err return @@ -159,16 +198,37 @@ func (proxy *Proxy) server() { "upstream": proxy.Upstream, }).Info("Accepted client") - upstream, err := net.Dial("tcp", proxy.Upstream) - if err != nil { - logrus.WithFields(logrus.Fields{ - "name": proxy.Name, - "client": client.RemoteAddr(), - "proxy": proxy.Listen, - "upstream": proxy.Upstream, - }).Error("Unable to open connection to upstream") - client.Close() - continue + if proxy.TLS != nil { + clientConfig := &tls.Config{InsecureSkipVerify: true} + upstreamTLS, err := tls.Dial("tcp", proxy.Upstream, clientConfig) + + if err != nil { + logrus.WithFields(logrus.Fields{ + "name": proxy.Name, + "client": client.RemoteAddr(), + "proxy": proxy.Listen, + "upstream": proxy.Upstream, + }).Error("Unable to open connection to upstream") + client.Close() + continue + } + upstream = upstreamTLS + + } else { + upstreamPlain, err := net.Dial("tcp", proxy.Upstream) + + if err != nil { + logrus.WithFields(logrus.Fields{ + "name": proxy.Name, + "client": client.RemoteAddr(), + "proxy": proxy.Listen, + "upstream": proxy.Upstream, + }).Error("Unable to open connection to upstream") + client.Close() + continue + } + + upstream = upstreamPlain } name := client.RemoteAddr().String() @@ -176,6 +236,7 @@ func (proxy *Proxy) server() { proxy.connections.list[name+"upstream"] = upstream proxy.connections.list[name+"downstream"] = client proxy.connections.Unlock() + proxy.Toxics.StartLink(name+"upstream", client, upstream, stream.Upstream) proxy.Toxics.StartLink(name+"downstream", upstream, client, stream.Downstream) } diff --git a/proxy_collection.go b/proxy_collection.go index 3070f5e7..1ad7dcc0 100644 --- a/proxy_collection.go +++ b/proxy_collection.go @@ -99,6 +99,8 @@ func (collection *ProxyCollection) PopulateJson(data io.Reader) ([]*Proxy, error proxy.Listen = p.Listen proxy.Upstream = p.Upstream + proxy.TLS = p.TLS + err = collection.AddOrReplace(proxy, *p.Enabled) if err != nil { break From a56cb053df09820f0e9b62089ff098e285b0f919 Mon Sep 17 00:00:00 2001 From: Gregor Pogacnik <1640719+fiksn@users.noreply.github.com> Date: Sun, 16 Feb 2020 14:21:31 +0100 Subject: [PATCH 2/4] Allow transparent TLS proxying --- README.md | 2 + TLS.md | 112 ++++++++++++++++++++++++++++++++++++++++++++ proxy.go | 138 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 247 insertions(+), 5 deletions(-) create mode 100644 TLS.md diff --git a/README.md b/README.md index 208c5230..3d7a727c 100644 --- a/README.md +++ b/README.md @@ -332,6 +332,8 @@ An example `config/toxiproxy.json` with the experimental TLS feature: ] ``` +For more details about TLS please check [TLS.md](./TLS.md). + Use ports outside the ephemeral port range to avoid random port conflicts. It's `32,768` to `61,000` on Linux by default, see `/proc/sys/net/ipv4/ip_local_port_range`. diff --git a/TLS.md b/TLS.md new file mode 100644 index 00000000..636be362 --- /dev/null +++ b/TLS.md @@ -0,0 +1,112 @@ +# TLS + +Using Toxiproxy with TLS presents its own challenges. +There are multiple ways how to use Toxiproxy in such a set-up. + +## Plain-connection + +That means Toxiproxy will just act as a TCP proxy. No patches are necessary. +TLS handshake will still be performed with actual endpoint. Thus Toxiproxy will +not be able to see (plain-text) traffic but may still apply toxic stuff (like delays) to the flow. + +Example `config/toxiproxy.json` +```json +[ + { + "name": "quasissl", + "listen": "[::]:443", + "upstream": "www.arnes.si:443", + "enabled": true + } +] +``` + +In this case you need to make sure the hostname (www.arnes.si in the example) +points to Toxiproxy IP. You could use hosts file for that with an entry like + +``` +127.0.0.1 www.arnes.si +``` + +but that isn't really the best option. A more scalable solution would be to change your DNS server to return fake responses. +Easiest is probably [Coredns](https://coredns.io) with rewrite plugin. + +## TLS connection with static certificate + +In this mode patched Toxiproxy will terminate the TLS connection and always return the configured certificate. + +Example `config/toxiproxy.json` +```json +[ + { + "name": "ssl", + "listen": "[::]:443", + "upstream": "www.arnes.si:443", + "enabled": true, + "tls": { + "cert": "./cert.crt", + "key": "./cert.key" + } + } +] +``` + +In this case users will configure different hostname - say toxiproxy.mydomain.org instead of www.arnes.si. If you have +proper X.509 certificate for toxiproxy.mydomain.org (for instance through [Let's Encrypt](https://letsencrypt.org)) everything +will behave fine. + +TLS section has an additional option: +"verifyUpstream" that is by default set to false. That is if we are already performing a Man-In-The-Middle attack it doesn't really make much +sense to be cautious about the upstream doing something similar. But you can always do something like: + +```json +[ + { + "name": "ssl", + "listen": "[::]:443", + "upstream": "www.arnes.si:443", + "enabled": true, + "tls": { + "cert": "./cert.crt", + "key": "./cert.key", + "verifyUpstream": true + } + } +] +``` + +## Dynamic certificates based on SNI + +In this mode patched Toxiproxy will observe what the hostname was in the request and use the given certificate as a CA to sign the new (dummy) certificate +that matches this hostname. Currently it will generate 2048 bit RSA keypair for that purpose. + +This mode is very similar to the first one (except that Toxiproxy is doing the TLS termination and can see plain-text traffic). You centrally enable transparent +proxying through Toxiproxy this way. + +An example `config/toxiproxy.json`: + +```json +[ + { + "name": "ssl", + "listen": "[::]:443", + "upstream": "www.arnes.si:443", + "enabled": true, + "tls": { + "cert": "./cert.crt", + "key": "./cert.key", + "isCA": true + } + } +] + +Here you need to alter DNS responses and additionally also configure the given CA cert (still passed in the configuration as cert/key) as trusted on all machines +that will be connecting to Toxiproxy. + +When isCA is true Toxiproxy will verify that cert.crt is actually a CA certificate (but you can always create a self-signed one of course). + +It is also possible to use "verifyUpstream" setting in this mode. + +## Notes + +Note that currently there is no option that Toxiproxy would terminate TLS connection and make a plain-text connection to the upstream as (for now) there is no use-case for it. diff --git a/proxy.go b/proxy.go index 63961889..f24c6383 100644 --- a/proxy.go +++ b/proxy.go @@ -2,9 +2,17 @@ package toxiproxy import ( "crypto/rand" + "crypto/rsa" "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" "errors" + "fmt" + "io/ioutil" + "math/big" "sync" + "time" "github.com/Shopify/toxiproxy/stream" "github.com/sirupsen/logrus" @@ -30,6 +38,7 @@ type Proxy struct { started chan error + caCert *tls.Certificate tomb tomb.Tomb connections ConnectionList Toxics *ToxicCollection `json:"-"` @@ -38,6 +47,10 @@ type Proxy struct { type TlsData struct { Cert string `json:"cert"` Key string `json:"key"` + // When the cert and key represent a CA, this can is used to dynamically sign fake certificates created with proper CN + IsCA bool `json:"isCA",omitempty` + // By default this is false (we are doing MITM attack so why bother with upstream certificate check) + VerifyUpstream bool `json:"verifyUpstream",omitempty` } type ConnectionList struct { @@ -97,6 +110,100 @@ func (proxy *Proxy) Stop() { stop(proxy) } +// Called for each new connection +func (proxy *Proxy) getCertificate(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { + var name string + + if hello.ServerName == "" { + name = "default" + } else { + name = hello.ServerName + } + + logrus.WithFields(logrus.Fields{ + "proxy": proxy.Name, + "serverName": name, + }).Info("getCertificate called") + + if proxy.caCert == nil { + return nil, errors.New("No CA certificate found") + } + + // Dynamically create new cert based on SNI + cert, err := createCertificate(*proxy.caCert, name) + if err != nil { + logrus.WithFields(logrus.Fields{ + "proxy": proxy.Name, + "serverName": name, + }).Errorf("created returned an error %s", err) + + return nil, err + } + return cert, nil +} + +// Ensure the given file is a CA certificate +func ensureCaCert(file string) error { + certFile, err := ioutil.ReadFile(file) + if err != nil { + return err + } + + block, _ := pem.Decode(certFile) + + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return err + } + + if cert.KeyUsage&x509.KeyUsageCertSign != x509.KeyUsageCertSign && !cert.IsCA { + return errors.New(fmt.Sprintf("The given certificate is not a CA cert - usage %d, isCA %t", cert.KeyUsage, cert.IsCA)) + } + + return nil +} + +// Utility function to create new certificate with given common name signed with our CA +func createCertificate(caTls tls.Certificate, commonName string) (*tls.Certificate, error) { + cert := &x509.Certificate{ + SerialNumber: big.NewInt(1337), + Subject: pkix.Name{ + Organization: []string{"Toxiproxy"}, + CommonName: commonName, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + IsCA: false, + SubjectKeyId: []byte{1, 2, 3, 4, 6}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + } + + priv, _ := rsa.GenerateKey(rand.Reader, 2048) + pub := &priv.PublicKey + + ca, err := x509.ParseCertificate(caTls.Certificate[0]) + if err != nil { + return nil, err + } + + certBlock, err := x509.CreateCertificate(rand.Reader, cert, ca, pub, caTls.PrivateKey) + if err != nil { + return nil, err + } + + newCert, err := tls.X509KeyPair( + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certBlock}), + pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}), + ) + + if err != nil { + return nil, err + } + + return &newCert, nil +} + // server runs the Proxy server, accepting new clients and creating Links to // connect them to upstreams. func (proxy *Proxy) server() { @@ -104,20 +211,33 @@ func (proxy *Proxy) server() { ln net.Listener err error upstream net.Conn + config tls.Config ) + // Logging if proxy.TLS != nil { logrus.WithFields(logrus.Fields{ - "proxy": proxy.Name, - "cert": proxy.TLS.Cert, - "key": proxy.TLS.Key, + "proxy": proxy.Name, + "cert": proxy.TLS.Cert, + "key": proxy.TLS.Key, + "isCA": proxy.TLS.IsCA, + "verifyUpstream": proxy.TLS.VerifyUpstream, }).Info("TLS certificates were specified") + + if proxy.TLS.IsCA { + err := ensureCaCert(proxy.TLS.Cert) + if err != nil { + proxy.started <- err + return + } + } } else { logrus.WithFields(logrus.Fields{ "proxy": proxy.Name, }).Info("TLS certificates were NOT specified") } + // Action if proxy.TLS != nil { cert, err := tls.LoadX509KeyPair(proxy.TLS.Cert, proxy.TLS.Key) if err != nil { @@ -125,8 +245,16 @@ func (proxy *Proxy) server() { return } - config := tls.Config{Certificates: []tls.Certificate{cert}} + if proxy.TLS.IsCA { + config = tls.Config{GetCertificate: proxy.getCertificate} + proxy.caCert = &cert + } else { + config = tls.Config{Certificates: []tls.Certificate{cert}} + proxy.caCert = nil + } + config.Rand = rand.Reader + ln, err = tls.Listen("tcp", proxy.Listen, &config) } else { ln, err = net.Listen("tcp", proxy.Listen) @@ -199,7 +327,7 @@ func (proxy *Proxy) server() { }).Info("Accepted client") if proxy.TLS != nil { - clientConfig := &tls.Config{InsecureSkipVerify: true} + clientConfig := &tls.Config{InsecureSkipVerify: !proxy.TLS.VerifyUpstream} upstreamTLS, err := tls.Dial("tcp", proxy.Upstream, clientConfig) if err != nil { From 4209112cfd39385350d380ccf8ece0de0074d4f3 Mon Sep 17 00:00:00 2001 From: nothinux Date: Mon, 20 May 2019 16:20:53 +0700 Subject: [PATCH 3/4] update docs --- client/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/README.md b/client/README.md index a078f872..d498b02f 100644 --- a/client/README.md +++ b/client/README.md @@ -39,7 +39,10 @@ client := toxiproxy.NewClient("localhost:8474") You can then create a new proxy using the client: ```go -proxy := client.CreateProxy("redis", "localhost:26379", "localhost:6379") +proxy, err := client.CreateProxy("redis", "localhost:26379", "localhost:6379") +if err != nil { + panic(err) +} ``` For large amounts of proxies, they can also be created using a configuration file: From 029d4d78a5dbe727affd7aea0f4360d17860dc75 Mon Sep 17 00:00:00 2001 From: Gregor Pogacnik <1640719+fiksn@users.noreply.github.com> Date: Mon, 17 Feb 2020 16:40:35 +0100 Subject: [PATCH 4/4] Better TLS docs --- TLS.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/TLS.md b/TLS.md index 636be362..7d28ec45 100644 --- a/TLS.md +++ b/TLS.md @@ -7,7 +7,7 @@ There are multiple ways how to use Toxiproxy in such a set-up. That means Toxiproxy will just act as a TCP proxy. No patches are necessary. TLS handshake will still be performed with actual endpoint. Thus Toxiproxy will -not be able to see (plain-text) traffic but may still apply toxic stuff (like delays) to the flow. +not be able to see (plain-text) traffic but may still apply toxics (like delays) to the flow. Example `config/toxiproxy.json` ```json @@ -29,7 +29,9 @@ points to Toxiproxy IP. You could use hosts file for that with an entry like ``` but that isn't really the best option. A more scalable solution would be to change your DNS server to return fake responses. -Easiest is probably [Coredns](https://coredns.io) with rewrite plugin. +Easiest is probably [Coredns](https://coredns.io) with rewrite plugin. + +Other option is a transparent proxy to forward specific traffic via iptables/netfilter rules. ## TLS connection with static certificate @@ -80,8 +82,7 @@ sense to be cautious about the upstream doing something similar. But you can alw In this mode patched Toxiproxy will observe what the hostname was in the request and use the given certificate as a CA to sign the new (dummy) certificate that matches this hostname. Currently it will generate 2048 bit RSA keypair for that purpose. -This mode is very similar to the first one (except that Toxiproxy is doing the TLS termination and can see plain-text traffic). You centrally enable transparent -proxying through Toxiproxy this way. +This mode is very similar to the first one (except that Toxiproxy is doing the TLS termination and can see plain-text traffic). An example `config/toxiproxy.json`: @@ -100,10 +101,14 @@ An example `config/toxiproxy.json`: } ] -Here you need to alter DNS responses and additionally also configure the given CA cert (still passed in the configuration as cert/key) as trusted on all machines -that will be connecting to Toxiproxy. +Here you need to do something similar to option number 1 and additionally also configure the given CA cert (still passed in the configuration as cert/key) as trusted on all machines +that will be connecting to Toxiproxy. Benefit is that you can centrally configure the interception rules (no need to change endpoints). + +You could use something like [SNIproxy](https://github.com/dlundquist/sniproxy) in front which makes it easier to just forward everything to the proxy and then route just specific +stuff through Toxiproxy. -When isCA is true Toxiproxy will verify that cert.crt is actually a CA certificate (but you can always create a self-signed one of course). +When isCA is true Toxiproxy will verify that cert.crt is actually a CA certificate (but you can always create a self-signed one of course). For now encrypted private key is not supported +(so be careful). It is also possible to use "verifyUpstream" setting in this mode.