From 74ea5092dd2161c774cfcc17c880d2005f726229 Mon Sep 17 00:00:00 2001 From: Paulo Sousa Date: Wed, 16 Oct 2024 18:30:39 -0300 Subject: [PATCH 1/3] feat: add proxy protocol support for svc loadbalancer type --- pkg/k8s/k8s.go | 59 +++++++++++++++++++------ pkg/k8s/k8s_test.go | 104 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+), 14 deletions(-) diff --git a/pkg/k8s/k8s.go b/pkg/k8s/k8s.go index 5b122bbc..44a1730d 100644 --- a/pkg/k8s/k8s.go +++ b/pkg/k8s/k8s.go @@ -38,6 +38,9 @@ const ( defaultHTTPSHostNetworkPort = int32(443) defaultHTTPSPortName = "https" + defaultProxyProtocolHTTPPortName = "proxy-http" + defaultProxyProtocolHTTPSPortName = "proxy-https" + defaultCacheVolumeExtraSize = float64(1.05) curlProbeCommand = "curl -m%d -kfsS -o /dev/null %s" @@ -233,20 +236,7 @@ func NewService(n *v1alpha1.Nginx) *corev1.Service { Annotations: annotations, }, Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: defaultHTTPPortName, - Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromString(defaultHTTPPortName), - Port: int32(80), - }, - { - Name: defaultHTTPSPortName, - Protocol: corev1.ProtocolTCP, - TargetPort: intstr.FromString(defaultHTTPSPortName), - Port: int32(443), - }, - }, + Ports: fillPorts(*n, nginxService(n)), Selector: labelSelector, LoadBalancerIP: lbIP, Type: nginxService(n), @@ -260,6 +250,47 @@ func NewService(n *v1alpha1.Nginx) *corev1.Service { return &service } +func fillPorts(n v1alpha1.Nginx, t corev1.ServiceType) []corev1.ServicePort { + if n.Spec.PodTemplate.Ports != nil && t == corev1.ServiceTypeLoadBalancer { + ports := make([]corev1.ServicePort, 0) + for _, port := range n.Spec.PodTemplate.Ports { + if port.Name == defaultProxyProtocolHTTPPortName { + ports = append(ports, corev1.ServicePort{ + Name: defaultProxyProtocolHTTPPortName, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromString(defaultProxyProtocolHTTPPortName), + Port: int32(80), + }) + } + if port.Name == defaultProxyProtocolHTTPSPortName { + ports = append(ports, corev1.ServicePort{ + Name: defaultProxyProtocolHTTPSPortName, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromString(defaultProxyProtocolHTTPSPortName), + Port: int32(443), + }) + } + } + if len(ports) > 0 { + return ports + } + } + return []corev1.ServicePort{ + { + Name: defaultHTTPPortName, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromString(defaultHTTPPortName), + Port: int32(80), + }, + { + Name: defaultHTTPSPortName, + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromString(defaultHTTPSPortName), + Port: int32(443), + }, + } +} + func nginxService(n *v1alpha1.Nginx) corev1.ServiceType { if n == nil || n.Spec.Service == nil { return corev1.ServiceTypeClusterIP diff --git a/pkg/k8s/k8s_test.go b/pkg/k8s/k8s_test.go index eb9df0ed..44a689de 100644 --- a/pkg/k8s/k8s_test.go +++ b/pkg/k8s/k8s_test.go @@ -1611,6 +1611,110 @@ func TestNewService(t *testing.T) { }, }, }, + { + name: "using proxy protocol with LB svc type", + nginx: func() v1alpha1.Nginx { + n := nginxWithService() + n.Spec.Service.Type = corev1.ServiceTypeLoadBalancer + n.Spec.PodTemplate.Ports = []corev1.ContainerPort{ + { + Name: defaultProxyProtocolHTTPPortName, + }, + { + Name: defaultProxyProtocolHTTPSPortName, + }, + } + return n + }(), + want: &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nginx-service", + Namespace: "default", + Labels: map[string]string{ + "nginx.tsuru.io/resource-name": "my-nginx", + "nginx.tsuru.io/app": "nginx", + }, + Annotations: map[string]string{}, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "proxy-http", + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromString("proxy-http"), + Port: int32(80), + }, + { + Name: "proxy-https", + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromString("proxy-https"), + Port: int32(443), + }, + }, + Selector: map[string]string{ + "nginx.tsuru.io/resource-name": "my-nginx", + "nginx.tsuru.io/app": "nginx", + }, + Type: corev1.ServiceTypeLoadBalancer, + }, + }, + }, + { + name: "using default ports with clusterIP svc type", + nginx: func() v1alpha1.Nginx { + n := nginxWithService() + n.Spec.Service.Type = corev1.ServiceTypeClusterIP + n.Spec.PodTemplate.Ports = []corev1.ContainerPort{ + { + Name: defaultProxyProtocolHTTPPortName, + }, + { + Name: defaultProxyProtocolHTTPSPortName, + }, + } + return n + }(), + want: &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nginx-service", + Namespace: "default", + Labels: map[string]string{ + "nginx.tsuru.io/resource-name": "my-nginx", + "nginx.tsuru.io/app": "nginx", + }, + Annotations: map[string]string{}, + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromString("http"), + Port: int32(80), + }, + { + Name: "https", + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromString("https"), + Port: int32(443), + }, + }, + Selector: map[string]string{ + "nginx.tsuru.io/resource-name": "my-nginx", + "nginx.tsuru.io/app": "nginx", + }, + Type: corev1.ServiceTypeClusterIP, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 863461b647d2258b793ff47a5416b4858057d097 Mon Sep 17 00:00:00 2001 From: Paulo Sousa Date: Thu, 17 Oct 2024 13:55:33 -0300 Subject: [PATCH 2/3] tests: fix certs on integration (new nginx with minimum size >1024 for RSA) --- examples/tls.yaml | 68 +++++++++++++++++++++++++--------------- test/integration_test.go | 4 +-- 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/examples/tls.yaml b/examples/tls.yaml index 164f0e59..47df2ff9 100644 --- a/examples/tls.yaml +++ b/examples/tls.yaml @@ -78,7 +78,7 @@ stringData: -----END PRIVATE KEY----- --- # TLS certificate generated with: -# go run $(go env GOROOT)/src/crypto/tls/generate_cert.go -start-date 'Jan 1 00:00:00 2021' -duration 87600h -host www.example.com -rsa-bits 1024 +# go run $(go env GOROOT)/src/crypto/tls/generate_cert.go -start-date 'Jan 1 00:00:00 2021' -duration 87600h -host www.example.com -rsa-bits 2048 apiVersion: v1 kind: Secret metadata: @@ -87,32 +87,50 @@ type: kubernetes.io/tls stringData: tls.crt: |- -----BEGIN CERTIFICATE----- - MIIB+jCCAWOgAwIBAgIQRLwc7m5PNh3qNWW9jbWIPzANBgkqhkiG9w0BAQsFADAS + MIIC/zCCAeegAwIBAgIQJKzZzFX2lwtGlsY/YDXhBDANBgkqhkiG9w0BAQsFADAS MRAwDgYDVQQKEwdBY21lIENvMB4XDTIxMDEwMTAwMDAwMFoXDTMwMTIzMDAwMDAw - MFowEjEQMA4GA1UEChMHQWNtZSBDbzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC - gYEAuZKXvIBthYPiXh1hBY8hJI8/uOLjushGDklmjI25Qfzs8TH6kPw647JXMMtr - 5afvgg9oMpJYeoVkBXL21iFe/yba/scyVb6Q9gsPtxppXv1m3SsqVqPu7SQtnvxP - ikBKiz76M7bl35do/M5j9iaKqzY0qhhcQBtGWkppN/JamvUCAwEAAaNRME8wDgYD - VR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAw - GgYDVR0RBBMwEYIPd3d3LmV4YW1wbGUuY29tMA0GCSqGSIb3DQEBCwUAA4GBAFpJ - HTqOKzhBBYeJsssorvRI09Vxm5iwVciCdUchyGT2VNZmkcuAlk40u9xTbfHz5iPX - Q6FcCxtr8C4sm9z5vLLDzzYVugQNosWoKMEOaw68PTG5OkNQUWOOCy0kSl9EG2wi - K0PSs3CqiW8q1wSgRYAQwZ+jRvl3QLiCdDQAYEPN + MFowEjEQMA4GA1UEChMHQWNtZSBDbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC + AQoCggEBAJ6xociK7Chs9MgDjuqs1vxSp1KYr8f70bZkLPDm3e4lqAkqTm8nBoN3 + BkIxUJrc/MiQdOrvJ4wKGc2YGD5vt9+F1uRMcDOWWBKoh1m+eHqwme7XyZkIPfD5 + tBTwGGVOuBHl1i158BPa6w2QQqdxh9sTJRYzusAX3BDEEBTV0Tck97us8CEvIEiy + U39Ok+40K+4T4wOXPKyFQKD1MwucNkfoFJedoDV1eogXnznPWbDQuZdHLNLXECBm + kcyQyvMuNKJ8ALxZpdTanEdiRVgFKFxCOAcxwNy5+7tcw6mLFL40YujnyLOonFsX + 1/2kSXytlCYOZSwgJZiaOuLpERxwcHECAwEAAaNRME8wDgYDVR0PAQH/BAQDAgWg + MBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwGgYDVR0RBBMwEYIP + d3d3LmV4YW1wbGUuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQBzSs3PAeowIlN9j28L + F1zg5Xh4GdiBc6sWofbmv24GVFTKwJovJguVA9PSpDj1kU2mrkoOrsNjxIv+UN4Y + Tf2aSvjVj78jtsl5CoRx5G79KdjCFIhNLd38tl6dA5PIgfe98K9eTsNMLnEZ5mcp + iMPrxX/8YYF7sdi9EcJJYz3p45VkqDh1sUC/s+M/onNbFn70HzDlEOez7Kdan2Hp + C3/bCKx0f46z2GtWbslaZ9DSx/kLrjfSSgRQDExhmX2bCCRU1W5jmWXIjOHJQtvp + sTEzjnpdBDLcukzwvkUSXJlZPghYFyY7iZpWbZysErtl0PY6B8dNP7pzqXgyfRCc + bSW2 -----END CERTIFICATE----- tls.key: |- -----BEGIN PRIVATE KEY----- - MIICeQIBADANBgkqhkiG9w0BAQEFAASCAmMwggJfAgEAAoGBALmSl7yAbYWD4l4d - YQWPISSPP7ji47rIRg5JZoyNuUH87PEx+pD8OuOyVzDLa+Wn74IPaDKSWHqFZAVy - 9tYhXv8m2v7HMlW+kPYLD7caaV79Zt0rKlaj7u0kLZ78T4pASos++jO25d+XaPzO - Y/Ymiqs2NKoYXEAbRlpKaTfyWpr1AgMBAAECgYEAnSbzsQLhvQvvC9y9aUPIwXk+ - cYhTar5n9XlbivQKS0OyNKAqndW82U8SX5Qlj5skTXRWvV0J6NhF2Ad/2d7w3pn0 - eWYZ0APeVKmIfqFytjVZEKx9kcjE//ADOF96I9Q4xkZF0E+PTS4eBpGdukVUqC+w - kEYwlaPzrVvS5fgjg4ECQQDUfSzORg8BKt6darM2dD5yIeYwEWqhxS+M2wb3Wy9s - J9ATm2n4fxX+ebu+YHs2tEcSEiCjZmGZC6I4wHCPsje9AkEA35JxoXP4ObUFJP/0 - 6w4Es7dUbNo8OGBG4FEGFRGQp/sf/LCyNo15hBIfUX4tHgFkZdoZw3tJAJJtKZad - hpenmQJBAKsTTKuWRCZQeYKLkVXaKRyCpm002ID93Nau0OD9jSSquR/QFHV4/CAx - qjIQQ6M7aazm41gq6ZDqiOTgKr/GYJECQQC25CmoliH9syPePLSyhOk7y0F5Op4A - 0GGqpx1+prvbbwefwzxGfD1n2RtdowZtgV4MI/rnezchDKfYWanLrMwhAkEAvosv - mbPtM41dl2mf6vxnop4PxA4nOX+DM0A2sWtokEKfZiG4t87WzbrkA0uvYgBJahNI - 2gDRg0Q++7eWC3uDqA== + MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCesaHIiuwobPTI + A47qrNb8UqdSmK/H+9G2ZCzw5t3uJagJKk5vJwaDdwZCMVCa3PzIkHTq7yeMChnN + mBg+b7ffhdbkTHAzllgSqIdZvnh6sJnu18mZCD3w+bQU8BhlTrgR5dYtefAT2usN + kEKncYfbEyUWM7rAF9wQxBAU1dE3JPe7rPAhLyBIslN/TpPuNCvuE+MDlzyshUCg + 9TMLnDZH6BSXnaA1dXqIF585z1mw0LmXRyzS1xAgZpHMkMrzLjSifAC8WaXU2pxH + YkVYBShcQjgHMcDcufu7XMOpixS+NGLo58izqJxbF9f9pEl8rZQmDmUsICWYmjri + 6REccHBxAgMBAAECggEAeMWO7mRrncht8dpjZc3sP1NX69Av1u6ah0htu1i4BYm6 + /F6qGoWEu4BVzwJg6brM1oa5tCIV4sm9IW5i5QF42i+tf0l9CoWcuXZwq6f9YMIO + JGstDb3sBfzDRDwFFPR3uYdFHI70Wuf8gvILCAuq7NsBGoh7071UyZ2s725aLrBM + h45LaH5c0+V0xSoSxi0l/0nzaqEg0Hl2/elaiZ2zT5IBBKVumdpr7cKL3uo6polB + zpgjs787Z9aaOBb/qECRGYs4QS2kyTtdyEmgUT7hMiBmOh3+1NXVl4aGAhpK28PV + mK4AtBWYxiQvvNZWSW4Clk1XUTT6ogBBsQx5cL900QKBgQDASDMajU03S72KWBvM + YwS1HWIZDU5diK8lcRw6aPc7YRGrEIpAADRahKLXqhD2l3kAj1yIrkp2OaaQsVeL + hBnOOt/djCohQQ5txKMnDFs0moy1/b/Iut3WEHDWvsDD5t5Lh3IlgNK85KGrbXAs + s9P2cAVR7aMYjLkbMnkX2e5XTwKBgQDTSA8esclzdSHoxx6eUPsMPKq2X2Eu37P8 + jteiaSgbNfo8U1NAWQOVsh7ZangkjtIwHGq+CGgkgqbII8ddNSPqGYd5X/4Mf24d + NHcXwIJq6bcwjqtOp9Idl49yQyIfIBBCOeDCvm8R+FsVivpiTa7+aPu4xZSUiQAg + hjb3kerMPwKBgD8nN4mwAT15Je7LU44Es6Yc2Xxq72MvIn+EsAXLNfCNyDTMQOwW + ZRuZ1rZIyDuFsptperX1sMU12CxL4voS0Dr5AVAN2kSDzo5k+EU5/LbBUBfyGMx/ + +550VetiSYPOTxK1kNsafZzG5wab8izsf2DAmC65g0jr51w0bqtTy8FnAoGBAIKF + U4Hcxtxike4hhFK9KDmpCLTdO0YfwCrXe3LDj5kvwUSfHSD1XHePr+sUEOBUspgy + FecLfnp/k9Sz9aco+uTVnJFosaK63iWAkcnEjUzK0PZYTI+ssOrDBETMIDomLqUa + 6DSmPCtXx70JPCCaaO6VoFvHA0C3WbK99xr9s5U1AoGAV+nyUxsoiX/OXHEvp5R8 + 8EPPm8B/lUWlVTLW0p3jpeG2R8tB/qh33M3MJmUjf5LBZOvbNBQPnVXNfWeo4p8N + gExPrJ6FMR5uzQ/rwc2Dhx3lxB/f2QaOhs8Zn/Nv8k6vwGxqrEG5T6IRmWp4It6u + mKlDHmDO7rr7LOAHAAd3NtA= -----END PRIVATE KEY----- diff --git a/test/integration_test.go b/test/integration_test.go index 29528f7c..7c1a1a0e 100644 --- a/test/integration_test.go +++ b/test/integration_test.go @@ -88,11 +88,11 @@ func Test_Operator(t *testing.T) { }{ { filename: "/etc/nginx/certs/my-rsa-cert/tls.crt", - expectedSha256: "6a95f3b95972f4beae4b54918498817d95f3d8cfb766c053cae656839e392430", + expectedSha256: "256bfbf7960e7f2fa4af64701a64525e794ffc4d6eab3d3ef79a0580a2f92fa5", }, { filename: "/etc/nginx/certs/my-rsa-cert/tls.key", - expectedSha256: "f8a6da5db392519513597345931237dbb1a0a28fb89b7ce3b41562f988f9ef67", + expectedSha256: "68694230fe7bca079f5b83549289b6fc65d3009803cc9100156147bbe8d62ff5", }, { filename: "/etc/nginx/certs/my-ecdsa-cert/tls.crt", From 70170f6d26896669d17c328b98e23f5bf249a94e Mon Sep 17 00:00:00 2001 From: Paulo Sousa Date: Fri, 18 Oct 2024 14:24:10 -0300 Subject: [PATCH 3/3] k8s: fix fillPorts function signature --- pkg/k8s/k8s.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/k8s/k8s.go b/pkg/k8s/k8s.go index 44a1730d..4e7c2baa 100644 --- a/pkg/k8s/k8s.go +++ b/pkg/k8s/k8s.go @@ -236,7 +236,7 @@ func NewService(n *v1alpha1.Nginx) *corev1.Service { Annotations: annotations, }, Spec: corev1.ServiceSpec{ - Ports: fillPorts(*n, nginxService(n)), + Ports: fillPorts(n, nginxService(n)), Selector: labelSelector, LoadBalancerIP: lbIP, Type: nginxService(n), @@ -250,7 +250,7 @@ func NewService(n *v1alpha1.Nginx) *corev1.Service { return &service } -func fillPorts(n v1alpha1.Nginx, t corev1.ServiceType) []corev1.ServicePort { +func fillPorts(n *v1alpha1.Nginx, t corev1.ServiceType) []corev1.ServicePort { if n.Spec.PodTemplate.Ports != nil && t == corev1.ServiceTypeLoadBalancer { ports := make([]corev1.ServicePort, 0) for _, port := range n.Spec.PodTemplate.Ports {