Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add proxy protocol support for svc loadbalancer type #54

Merged
merged 3 commits into from
Oct 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 43 additions & 25 deletions examples/tls.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ stringData:
-----END PRIVATE KEY-----
---
# TLS certificate generated with:
# go run $(go env GOROOT)/src/crypto/tls/generate_cert.go -start-date 'Jan 1 00:00:00 2021' -duration 87600h -host www.example.com -rsa-bits 1024
# go run $(go env GOROOT)/src/crypto/tls/generate_cert.go -start-date 'Jan 1 00:00:00 2021' -duration 87600h -host www.example.com -rsa-bits 2048
apiVersion: v1
kind: Secret
metadata:
Expand All @@ -87,32 +87,50 @@ type: kubernetes.io/tls
stringData:
tls.crt: |-
-----BEGIN CERTIFICATE-----
MIIB+jCCAWOgAwIBAgIQRLwc7m5PNh3qNWW9jbWIPzANBgkqhkiG9w0BAQsFADAS
MIIC/zCCAeegAwIBAgIQJKzZzFX2lwtGlsY/YDXhBDANBgkqhkiG9w0BAQsFADAS
MRAwDgYDVQQKEwdBY21lIENvMB4XDTIxMDEwMTAwMDAwMFoXDTMwMTIzMDAwMDAw
MFowEjEQMA4GA1UEChMHQWNtZSBDbzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC
gYEAuZKXvIBthYPiXh1hBY8hJI8/uOLjushGDklmjI25Qfzs8TH6kPw647JXMMtr
5afvgg9oMpJYeoVkBXL21iFe/yba/scyVb6Q9gsPtxppXv1m3SsqVqPu7SQtnvxP
ikBKiz76M7bl35do/M5j9iaKqzY0qhhcQBtGWkppN/JamvUCAwEAAaNRME8wDgYD
VR0PAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAw
GgYDVR0RBBMwEYIPd3d3LmV4YW1wbGUuY29tMA0GCSqGSIb3DQEBCwUAA4GBAFpJ
HTqOKzhBBYeJsssorvRI09Vxm5iwVciCdUchyGT2VNZmkcuAlk40u9xTbfHz5iPX
Q6FcCxtr8C4sm9z5vLLDzzYVugQNosWoKMEOaw68PTG5OkNQUWOOCy0kSl9EG2wi
K0PSs3CqiW8q1wSgRYAQwZ+jRvl3QLiCdDQAYEPN
MFowEjEQMA4GA1UEChMHQWNtZSBDbzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
AQoCggEBAJ6xociK7Chs9MgDjuqs1vxSp1KYr8f70bZkLPDm3e4lqAkqTm8nBoN3
BkIxUJrc/MiQdOrvJ4wKGc2YGD5vt9+F1uRMcDOWWBKoh1m+eHqwme7XyZkIPfD5
tBTwGGVOuBHl1i158BPa6w2QQqdxh9sTJRYzusAX3BDEEBTV0Tck97us8CEvIEiy
U39Ok+40K+4T4wOXPKyFQKD1MwucNkfoFJedoDV1eogXnznPWbDQuZdHLNLXECBm
kcyQyvMuNKJ8ALxZpdTanEdiRVgFKFxCOAcxwNy5+7tcw6mLFL40YujnyLOonFsX
1/2kSXytlCYOZSwgJZiaOuLpERxwcHECAwEAAaNRME8wDgYDVR0PAQH/BAQDAgWg
MBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwGgYDVR0RBBMwEYIP
d3d3LmV4YW1wbGUuY29tMA0GCSqGSIb3DQEBCwUAA4IBAQBzSs3PAeowIlN9j28L
F1zg5Xh4GdiBc6sWofbmv24GVFTKwJovJguVA9PSpDj1kU2mrkoOrsNjxIv+UN4Y
Tf2aSvjVj78jtsl5CoRx5G79KdjCFIhNLd38tl6dA5PIgfe98K9eTsNMLnEZ5mcp
iMPrxX/8YYF7sdi9EcJJYz3p45VkqDh1sUC/s+M/onNbFn70HzDlEOez7Kdan2Hp
C3/bCKx0f46z2GtWbslaZ9DSx/kLrjfSSgRQDExhmX2bCCRU1W5jmWXIjOHJQtvp
sTEzjnpdBDLcukzwvkUSXJlZPghYFyY7iZpWbZysErtl0PY6B8dNP7pzqXgyfRCc
bSW2
-----END CERTIFICATE-----
tls.key: |-
-----BEGIN PRIVATE KEY-----
MIICeQIBADANBgkqhkiG9w0BAQEFAASCAmMwggJfAgEAAoGBALmSl7yAbYWD4l4d
YQWPISSPP7ji47rIRg5JZoyNuUH87PEx+pD8OuOyVzDLa+Wn74IPaDKSWHqFZAVy
9tYhXv8m2v7HMlW+kPYLD7caaV79Zt0rKlaj7u0kLZ78T4pASos++jO25d+XaPzO
Y/Ymiqs2NKoYXEAbRlpKaTfyWpr1AgMBAAECgYEAnSbzsQLhvQvvC9y9aUPIwXk+
cYhTar5n9XlbivQKS0OyNKAqndW82U8SX5Qlj5skTXRWvV0J6NhF2Ad/2d7w3pn0
eWYZ0APeVKmIfqFytjVZEKx9kcjE//ADOF96I9Q4xkZF0E+PTS4eBpGdukVUqC+w
kEYwlaPzrVvS5fgjg4ECQQDUfSzORg8BKt6darM2dD5yIeYwEWqhxS+M2wb3Wy9s
J9ATm2n4fxX+ebu+YHs2tEcSEiCjZmGZC6I4wHCPsje9AkEA35JxoXP4ObUFJP/0
6w4Es7dUbNo8OGBG4FEGFRGQp/sf/LCyNo15hBIfUX4tHgFkZdoZw3tJAJJtKZad
hpenmQJBAKsTTKuWRCZQeYKLkVXaKRyCpm002ID93Nau0OD9jSSquR/QFHV4/CAx
qjIQQ6M7aazm41gq6ZDqiOTgKr/GYJECQQC25CmoliH9syPePLSyhOk7y0F5Op4A
0GGqpx1+prvbbwefwzxGfD1n2RtdowZtgV4MI/rnezchDKfYWanLrMwhAkEAvosv
mbPtM41dl2mf6vxnop4PxA4nOX+DM0A2sWtokEKfZiG4t87WzbrkA0uvYgBJahNI
2gDRg0Q++7eWC3uDqA==
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCesaHIiuwobPTI
A47qrNb8UqdSmK/H+9G2ZCzw5t3uJagJKk5vJwaDdwZCMVCa3PzIkHTq7yeMChnN
mBg+b7ffhdbkTHAzllgSqIdZvnh6sJnu18mZCD3w+bQU8BhlTrgR5dYtefAT2usN
kEKncYfbEyUWM7rAF9wQxBAU1dE3JPe7rPAhLyBIslN/TpPuNCvuE+MDlzyshUCg
9TMLnDZH6BSXnaA1dXqIF585z1mw0LmXRyzS1xAgZpHMkMrzLjSifAC8WaXU2pxH
YkVYBShcQjgHMcDcufu7XMOpixS+NGLo58izqJxbF9f9pEl8rZQmDmUsICWYmjri
6REccHBxAgMBAAECggEAeMWO7mRrncht8dpjZc3sP1NX69Av1u6ah0htu1i4BYm6
/F6qGoWEu4BVzwJg6brM1oa5tCIV4sm9IW5i5QF42i+tf0l9CoWcuXZwq6f9YMIO
JGstDb3sBfzDRDwFFPR3uYdFHI70Wuf8gvILCAuq7NsBGoh7071UyZ2s725aLrBM
h45LaH5c0+V0xSoSxi0l/0nzaqEg0Hl2/elaiZ2zT5IBBKVumdpr7cKL3uo6polB
zpgjs787Z9aaOBb/qECRGYs4QS2kyTtdyEmgUT7hMiBmOh3+1NXVl4aGAhpK28PV
mK4AtBWYxiQvvNZWSW4Clk1XUTT6ogBBsQx5cL900QKBgQDASDMajU03S72KWBvM
YwS1HWIZDU5diK8lcRw6aPc7YRGrEIpAADRahKLXqhD2l3kAj1yIrkp2OaaQsVeL
hBnOOt/djCohQQ5txKMnDFs0moy1/b/Iut3WEHDWvsDD5t5Lh3IlgNK85KGrbXAs
s9P2cAVR7aMYjLkbMnkX2e5XTwKBgQDTSA8esclzdSHoxx6eUPsMPKq2X2Eu37P8
jteiaSgbNfo8U1NAWQOVsh7ZangkjtIwHGq+CGgkgqbII8ddNSPqGYd5X/4Mf24d
NHcXwIJq6bcwjqtOp9Idl49yQyIfIBBCOeDCvm8R+FsVivpiTa7+aPu4xZSUiQAg
hjb3kerMPwKBgD8nN4mwAT15Je7LU44Es6Yc2Xxq72MvIn+EsAXLNfCNyDTMQOwW
ZRuZ1rZIyDuFsptperX1sMU12CxL4voS0Dr5AVAN2kSDzo5k+EU5/LbBUBfyGMx/
+550VetiSYPOTxK1kNsafZzG5wab8izsf2DAmC65g0jr51w0bqtTy8FnAoGBAIKF
U4Hcxtxike4hhFK9KDmpCLTdO0YfwCrXe3LDj5kvwUSfHSD1XHePr+sUEOBUspgy
FecLfnp/k9Sz9aco+uTVnJFosaK63iWAkcnEjUzK0PZYTI+ssOrDBETMIDomLqUa
6DSmPCtXx70JPCCaaO6VoFvHA0C3WbK99xr9s5U1AoGAV+nyUxsoiX/OXHEvp5R8
8EPPm8B/lUWlVTLW0p3jpeG2R8tB/qh33M3MJmUjf5LBZOvbNBQPnVXNfWeo4p8N
gExPrJ6FMR5uzQ/rwc2Dhx3lxB/f2QaOhs8Zn/Nv8k6vwGxqrEG5T6IRmWp4It6u
mKlDHmDO7rr7LOAHAAd3NtA=
-----END PRIVATE KEY-----
59 changes: 45 additions & 14 deletions pkg/k8s/k8s.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ const (
defaultHTTPSHostNetworkPort = int32(443)
defaultHTTPSPortName = "https"

defaultProxyProtocolHTTPPortName = "proxy-http"
defaultProxyProtocolHTTPSPortName = "proxy-https"

defaultCacheVolumeExtraSize = float64(1.05)

curlProbeCommand = "curl -m%d -kfsS -o /dev/null %s"
Expand Down Expand Up @@ -233,20 +236,7 @@ func NewService(n *v1alpha1.Nginx) *corev1.Service {
Annotations: annotations,
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{
{
Name: defaultHTTPPortName,
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromString(defaultHTTPPortName),
Port: int32(80),
},
{
Name: defaultHTTPSPortName,
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromString(defaultHTTPSPortName),
Port: int32(443),
},
},
Ports: fillPorts(n, nginxService(n)),
Selector: labelSelector,
LoadBalancerIP: lbIP,
Type: nginxService(n),
Expand All @@ -260,6 +250,47 @@ func NewService(n *v1alpha1.Nginx) *corev1.Service {
return &service
}

func fillPorts(n *v1alpha1.Nginx, t corev1.ServiceType) []corev1.ServicePort {
if n.Spec.PodTemplate.Ports != nil && t == corev1.ServiceTypeLoadBalancer {
ports := make([]corev1.ServicePort, 0)
for _, port := range n.Spec.PodTemplate.Ports {
if port.Name == defaultProxyProtocolHTTPPortName {
ports = append(ports, corev1.ServicePort{
Name: defaultProxyProtocolHTTPPortName,
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromString(defaultProxyProtocolHTTPPortName),
Port: int32(80),
})
}
if port.Name == defaultProxyProtocolHTTPSPortName {
ports = append(ports, corev1.ServicePort{
Name: defaultProxyProtocolHTTPSPortName,
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromString(defaultProxyProtocolHTTPSPortName),
Port: int32(443),
})
}
}
if len(ports) > 0 {
return ports
}
}
return []corev1.ServicePort{
{
Name: defaultHTTPPortName,
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromString(defaultHTTPPortName),
Port: int32(80),
},
{
Name: defaultHTTPSPortName,
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromString(defaultHTTPSPortName),
Port: int32(443),
},
}
}

func nginxService(n *v1alpha1.Nginx) corev1.ServiceType {
if n == nil || n.Spec.Service == nil {
return corev1.ServiceTypeClusterIP
Expand Down
104 changes: 104 additions & 0 deletions pkg/k8s/k8s_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1611,6 +1611,110 @@ func TestNewService(t *testing.T) {
},
},
},
{
name: "using proxy protocol with LB svc type",
nginx: func() v1alpha1.Nginx {
n := nginxWithService()
n.Spec.Service.Type = corev1.ServiceTypeLoadBalancer
n.Spec.PodTemplate.Ports = []corev1.ContainerPort{
{
Name: defaultProxyProtocolHTTPPortName,
},
{
Name: defaultProxyProtocolHTTPSPortName,
},
}
return n
}(),
want: &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "my-nginx-service",
Namespace: "default",
Labels: map[string]string{
"nginx.tsuru.io/resource-name": "my-nginx",
"nginx.tsuru.io/app": "nginx",
},
Annotations: map[string]string{},
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{
{
Name: "proxy-http",
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromString("proxy-http"),
Port: int32(80),
},
{
Name: "proxy-https",
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromString("proxy-https"),
Port: int32(443),
},
},
Selector: map[string]string{
"nginx.tsuru.io/resource-name": "my-nginx",
"nginx.tsuru.io/app": "nginx",
},
Type: corev1.ServiceTypeLoadBalancer,
},
},
},
{
name: "using default ports with clusterIP svc type",
nginx: func() v1alpha1.Nginx {
n := nginxWithService()
n.Spec.Service.Type = corev1.ServiceTypeClusterIP
n.Spec.PodTemplate.Ports = []corev1.ContainerPort{
{
Name: defaultProxyProtocolHTTPPortName,
},
{
Name: defaultProxyProtocolHTTPSPortName,
},
}
return n
}(),
want: &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "my-nginx-service",
Namespace: "default",
Labels: map[string]string{
"nginx.tsuru.io/resource-name": "my-nginx",
"nginx.tsuru.io/app": "nginx",
},
Annotations: map[string]string{},
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{
{
Name: "http",
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromString("http"),
Port: int32(80),
},
{
Name: "https",
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromString("https"),
Port: int32(443),
},
},
Selector: map[string]string{
"nginx.tsuru.io/resource-name": "my-nginx",
"nginx.tsuru.io/app": "nginx",
},
Type: corev1.ServiceTypeClusterIP,
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
4 changes: 2 additions & 2 deletions test/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,11 +88,11 @@ func Test_Operator(t *testing.T) {
}{
{
filename: "/etc/nginx/certs/my-rsa-cert/tls.crt",
expectedSha256: "6a95f3b95972f4beae4b54918498817d95f3d8cfb766c053cae656839e392430",
expectedSha256: "256bfbf7960e7f2fa4af64701a64525e794ffc4d6eab3d3ef79a0580a2f92fa5",
},
{
filename: "/etc/nginx/certs/my-rsa-cert/tls.key",
expectedSha256: "f8a6da5db392519513597345931237dbb1a0a28fb89b7ce3b41562f988f9ef67",
expectedSha256: "68694230fe7bca079f5b83549289b6fc65d3009803cc9100156147bbe8d62ff5",
},
{
filename: "/etc/nginx/certs/my-ecdsa-cert/tls.crt",
Expand Down
Loading