From 334f4180061069b2e01446f0f5d86aa11e47e37a Mon Sep 17 00:00:00 2001 From: Sandhya Dasu Date: Sun, 5 Nov 2023 05:04:08 -0500 Subject: [PATCH] Adding ability to generate Corefile using LB IP addresses A new capability is being added to cloud platforms (starting with AWS, Azure and GCP) where the cloud LBs can be used but not the cloud DNS. So, in-cluster DNS is provided by a CoreDNS pod during install. The customer can optionally bring their own DNS after install complete. This commit adds the ability to generate a CoreDNS Corefile with entries for API and API-Int URLs when their corresponding LB IPs are provided. These LB IPs are not expected to change during the life of the cluster. --- cmd/corednsmonitor/corednsmonitor.go | 10 ++- cmd/runtimecfg/display.go | 13 +++- cmd/runtimecfg/render.go | 12 +++- pkg/config/node.go | 94 +++++++++++++++++++++------- pkg/monitor/corednsmonitor.go | 8 ++- pkg/monitor/dnsmasqmonitor.go | 2 +- pkg/monitor/dynkeepalived.go | 4 +- 7 files changed, 114 insertions(+), 29 deletions(-) diff --git a/cmd/corednsmonitor/corednsmonitor.go b/cmd/corednsmonitor/corednsmonitor.go index e5f8680a..cdff4e5c 100644 --- a/cmd/corednsmonitor/corednsmonitor.go +++ b/cmd/corednsmonitor/corednsmonitor.go @@ -55,8 +55,16 @@ func main() { if err != nil { return err } + cloudExtLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips") + if err != nil { + cloudExtLBIPs = []net.IP{} + } + cloudIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips") + if err != nil { + cloudIntLBIPs = []net.IP{} + } - return monitor.CorednsWatch(args[0], clusterConfigPath, args[1], args[2], apiVips, ingressVips, checkInterval) + return monitor.CorednsWatch(args[0], clusterConfigPath, args[1], args[2], apiVips, ingressVips, checkInterval, cloudExtLBIPs, cloudIntLBIPs) }, } rootCmd.PersistentFlags().StringP("cluster-config", "c", "", "Path to cluster-config ConfigMap to retrieve ControlPlane info") diff --git a/cmd/runtimecfg/display.go b/cmd/runtimecfg/display.go index 97134a63..aa5ede07 100644 --- a/cmd/runtimecfg/display.go +++ b/cmd/runtimecfg/display.go @@ -28,6 +28,8 @@ func init() { displayCmd.Flags().Uint16("lb-port", 9445, "Port where the API HAProxy LB will listen at") displayCmd.Flags().Uint16("stat-port", 29445, "Port where the HAProxy stats API will listen at") displayCmd.Flags().StringP("resolvconf-path", "r", "/etc/resolv.conf", "Optional path to a resolv.conf file to use to get upstream DNS servers") + displayCmd.Flags().IPSlice("cloud-ext-lb-ips", nil, "IP Addresses of Cloud External Load Balancers for OpenShift API") + displayCmd.Flags().IPSlice("cloud-int-lb-ips", nil, "IP Addresses of Cloud Internal Load Balancers for OpenShift API") rootCmd.AddCommand(displayCmd) } @@ -84,7 +86,16 @@ func runDisplay(cmd *cobra.Command, args []string) error { if err != nil { return err } - config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort) + + apiLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips") + if err != nil { + apiLBIPs = []net.IP{} + } + apiIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips") + if err != nil { + apiIntLBIPs = []net.IP{} + } + config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort, apiLBIPs, apiIntLBIPs) if err != nil { return err } diff --git a/cmd/runtimecfg/render.go b/cmd/runtimecfg/render.go index 528e831f..1d894956 100644 --- a/cmd/runtimecfg/render.go +++ b/cmd/runtimecfg/render.go @@ -33,6 +33,8 @@ func init() { renderCmd.Flags().Uint16("lb-port", 9445, "Port where the API HAProxy LB will listen at") renderCmd.Flags().Uint16("stat-port", 29445, "Port where the HAProxy stats API will listen at") renderCmd.Flags().StringP("resolvconf-path", "r", "/etc/resolv.conf", "Optional path to a resolv.conf file to use to get upstream DNS servers") + renderCmd.Flags().IPSlice("cloud-ext-lb-ips", nil, "IP Addresses of Cloud External Load Balancers for OpenShift API") + renderCmd.Flags().IPSlice("cloud-int-lb-ips", nil, "IP Addresses of Cloud Internal Load Balancers for OpenShift Internal API") rootCmd.AddCommand(renderCmd) } @@ -88,7 +90,15 @@ func runRender(cmd *cobra.Command, args []string) error { if err != nil { return err } - config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort) + apiLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips") + if err != nil { + apiLBIPs = []net.IP{} + } + apiIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips") + if err != nil { + apiIntLBIPs = []net.IP{} + } + config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort, apiLBIPs, apiIntLBIPs) if err != nil { return err } diff --git a/pkg/config/node.go b/pkg/config/node.go index 84ebb396..5de971d1 100644 --- a/pkg/config/node.go +++ b/pkg/config/node.go @@ -52,6 +52,10 @@ type Cluster struct { VIPNetmask int MasterAmount int64 NodeAddresses []NodeAddress + APILBIPs []string + APIIntLBIPs []string + CloudLBRecordType string + CloudLBEmptyType string } type Backend struct { @@ -417,37 +421,58 @@ func getNodeIpForRequestedIpStack(node v1.Node, filterIps []string, machineNetwo // apiPort: The port on which the k8s api listens. Should be 6443. // lbPort: The port on which haproxy listens. // statPort: The port on which the haproxy stats endpoint listens. -func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips, ingressVips []net.IP, apiPort, lbPort, statPort uint16) (node Node, err error) { - vipCount := 0 - if len(apiVips) > len(ingressVips) { - vipCount = len(apiVips) - } else { - vipCount = len(ingressVips) - } +// apiLBIPs: A list of External Load Balancer IPs for API. +// apiIntLBIPs: A list of Internal Load Balancer IPs for API. +func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips, ingressVips []net.IP, apiPort, lbPort, statPort uint16, apiLBIPs, apiIntLBIPs []net.IP) (node Node, err error) { nodes := []Node{} - var apiVip, ingressVip net.IP - for i := 0; i < vipCount; i++ { - if i < len(apiVips) { - apiVip = apiVips[i] + if len(apiIntLBIPs) == 0 { + // On-prem platforms + vipCount := 0 + if len(apiVips) > len(ingressVips) { + vipCount = len(apiVips) } else { - apiVip = nil + vipCount = len(ingressVips) } - if i < len(ingressVips) { - ingressVip = ingressVips[i] - } else { - ingressVip = nil + var apiVip, ingressVip net.IP + for i := 0; i < vipCount; i++ { + if i < len(apiVips) { + apiVip = apiVips[i] + } else { + apiVip = nil + } + if i < len(ingressVips) { + ingressVip = ingressVips[i] + } else { + ingressVip = nil + } + newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, apiVip, ingressVip, apiPort, lbPort, statPort, nil, nil) + if err != nil { + return Node{}, err + } + nodes = append(nodes, newNode) } - newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, apiVip, ingressVip, apiPort, lbPort, statPort) - if err != nil { - return Node{}, err + } else { + // Cloud Platforms with cloud LBs but no Cloud DNS + var apiLBIP, apiIntLBIP net.IP + for i := 0; i < len(apiIntLBIPs); i++ { + apiIntLBIP = apiIntLBIPs[i] + // For public clusters. Private clusters will not have External + // LBs so apiLBIPs could be empty. + if len(apiLBIPs) != 0 { + apiLBIP = apiLBIPs[i] + } + newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, nil, nil, 0, 0, 0, apiLBIP, apiIntLBIP) + if err != nil { + return Node{}, err + } + nodes = append(nodes, newNode) } - nodes = append(nodes, newNode) } nodes[0].Configs = &nodes return nodes[0], nil } -func getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip net.IP, ingressVip net.IP, apiPort, lbPort, statPort uint16) (node Node, err error) { +func getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip net.IP, ingressVip net.IP, apiPort, lbPort, statPort uint16, apiLBIP, apiIntLBIP net.IP) (node Node, err error) { clusterName, clusterDomain, err := GetClusterNameAndDomain(kubeconfigPath, clusterConfigPath) if err != nil { return node, err @@ -534,6 +559,24 @@ func getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, api StatPort: statPort, } + // For cloud platforms that need an in-cluster DNS solution, + // API and API-Int LB IPs are used to generate CoreDNS config + // The same DNS record type can be used for both entries for + // API and API-Int resolution. + node.Cluster.CloudLBRecordType = "A" + node.Cluster.CloudLBEmptyType = "AAAA" + if apiIntLBIP != nil { + node.Cluster.APIIntLBIPs = append(node.Cluster.APIIntLBIPs, apiIntLBIP.String()) + node.Cluster.CloudLBRecordType = "A" + node.Cluster.CloudLBEmptyType = "AAAA" + if apiIntLBIP.To4() == nil { + node.Cluster.CloudLBRecordType = "AAAA" + node.Cluster.CloudLBEmptyType = "A" + } + } + if apiLBIP != nil { + node.Cluster.APILBIPs = append(node.Cluster.APILBIPs, apiLBIP.String()) + } return node, err } @@ -694,3 +737,12 @@ func PopulateNodeAddresses(kubeconfigPath string, node *Node) { } } } + +func PopulateCloudLBIPAddresses(apiLBIPs, apiIntLBIPs []net.IP, node *Node) { + for i := 0; i < len(apiIntLBIPs); i++ { + node.Cluster.APIIntLBIPs = append(node.Cluster.APIIntLBIPs, apiIntLBIPs[i].String()) + } + for i := 0; i < len(apiLBIPs); i++ { + node.Cluster.APILBIPs = append(node.Cluster.APILBIPs, apiLBIPs[i].String()) + } +} diff --git a/pkg/monitor/corednsmonitor.go b/pkg/monitor/corednsmonitor.go index fcb80b9a..17304085 100644 --- a/pkg/monitor/corednsmonitor.go +++ b/pkg/monitor/corednsmonitor.go @@ -16,7 +16,7 @@ import ( const resolvConfFilepath string = "/var/run/NetworkManager/resolv.conf" -func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath string, apiVips, ingressVips []net.IP, interval time.Duration) error { +func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath string, apiVips, ingressVips []net.IP, interval time.Duration, apiLBIPs, apiIntLBIPs []net.IP) error { signals := make(chan os.Signal, 1) done := make(chan bool, 1) @@ -42,10 +42,14 @@ func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath strin if err != nil { return err } - newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, resolvConfFilepath, apiVips, ingressVips, 0, 0, 0) + newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, resolvConfFilepath, apiVips, ingressVips, 0, 0, 0, apiLBIPs, apiIntLBIPs) if err != nil { return err } + // Populate cloud LB IP addresses for platforms where the cloud LBs + // have already been configured + config.PopulateCloudLBIPAddresses(apiLBIPs, apiIntLBIPs, &newConfig) + config.PopulateNodeAddresses(kubeconfigPath, &newConfig) // There should never be 0 nodes in a functioning cluster. This means // we failed to populate the list, so we don't want to render. diff --git a/pkg/monitor/dnsmasqmonitor.go b/pkg/monitor/dnsmasqmonitor.go index 5371049a..bfde5d80 100644 --- a/pkg/monitor/dnsmasqmonitor.go +++ b/pkg/monitor/dnsmasqmonitor.go @@ -33,7 +33,7 @@ func DnsmasqWatch(kubeconfigPath, templatePath, cfgPath string, apiVips []net.IP return nil default: // We only care about the api vip and cluster domain here - config, err := config.GetConfig(kubeconfigPath, "", "/etc/resolv.conf", apiVips, apiVips, 0, 0, 0) + config, err := config.GetConfig(kubeconfigPath, "", "/etc/resolv.conf", apiVips, apiVips, 0, 0, 0, []net.IP{}, []net.IP{}) if err != nil { return err } diff --git a/pkg/monitor/dynkeepalived.go b/pkg/monitor/dynkeepalived.go index c143d870..62590429 100644 --- a/pkg/monitor/dynkeepalived.go +++ b/pkg/monitor/dynkeepalived.go @@ -355,7 +355,7 @@ func KeepalivedWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath st case desiredModeInfo := <-updateModeCh: - newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0) + newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0, []net.IP{}, []net.IP{}) if err != nil { return err } @@ -437,7 +437,7 @@ func KeepalivedWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath st } } } - newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0) + newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0, []net.IP{}, []net.IP{}) if err != nil { return err }