Adding ability to generate Corefile using LB IP addresses

A new capability is being added to cloud platforms (starting with AWS, Azure and GCP) where the cloud LBs can be used but not the cloud DNS. So, in-cluster DNS is provided by a CoreDNS pod during install. The customer can optionally bring their own DNS after install complete. This commit adds the ability to generate a CoreDNS Corefile with entries for API and API-Int URLs when their corresponding LB IPs are provided. These LB IPs are not expected to change during the life of the cluster.
openshift · Nov 16, 2023 · 334f418 · 334f418
1 parent d82b944
commit 334f418
Show file tree

Hide file tree

Showing 7 changed files with 114 additions and 29 deletions.
diff --git a/cmd/corednsmonitor/corednsmonitor.go b/cmd/corednsmonitor/corednsmonitor.go
@@ -55,8 +55,16 @@ func main() {
 			if err != nil {
 				return err
 			}
+			cloudExtLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips")
+			if err != nil {
+				cloudExtLBIPs = []net.IP{}
+			}
+			cloudIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips")
+			if err != nil {
+				cloudIntLBIPs = []net.IP{}
+			}
 
-			return monitor.CorednsWatch(args[0], clusterConfigPath, args[1], args[2], apiVips, ingressVips, checkInterval)
+			return monitor.CorednsWatch(args[0], clusterConfigPath, args[1], args[2], apiVips, ingressVips, checkInterval, cloudExtLBIPs, cloudIntLBIPs)
 		},
 	}
 	rootCmd.PersistentFlags().StringP("cluster-config", "c", "", "Path to cluster-config ConfigMap to retrieve ControlPlane info")

diff --git a/cmd/runtimecfg/display.go b/cmd/runtimecfg/display.go
@@ -28,6 +28,8 @@ func init() {
 	displayCmd.Flags().Uint16("lb-port", 9445, "Port where the API HAProxy LB will listen at")
 	displayCmd.Flags().Uint16("stat-port", 29445, "Port where the HAProxy stats API will listen at")
 	displayCmd.Flags().StringP("resolvconf-path", "r", "/etc/resolv.conf", "Optional path to a resolv.conf file to use to get upstream DNS servers")
+	displayCmd.Flags().IPSlice("cloud-ext-lb-ips", nil, "IP Addresses of Cloud External Load Balancers for OpenShift API")
+	displayCmd.Flags().IPSlice("cloud-int-lb-ips", nil, "IP Addresses of Cloud Internal Load Balancers for OpenShift API")
 	rootCmd.AddCommand(displayCmd)
 }
 
@@ -84,7 +86,16 @@ func runDisplay(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort)
+
+	apiLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips")
+	if err != nil {
+		apiLBIPs = []net.IP{}
+	}
+	apiIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips")
+	if err != nil {
+		apiIntLBIPs = []net.IP{}
+	}
+	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort, apiLBIPs, apiIntLBIPs)
 	if err != nil {
 		return err
 	}

diff --git a/cmd/runtimecfg/render.go b/cmd/runtimecfg/render.go
@@ -33,6 +33,8 @@ func init() {
 	renderCmd.Flags().Uint16("lb-port", 9445, "Port where the API HAProxy LB will listen at")
 	renderCmd.Flags().Uint16("stat-port", 29445, "Port where the HAProxy stats API will listen at")
 	renderCmd.Flags().StringP("resolvconf-path", "r", "/etc/resolv.conf", "Optional path to a resolv.conf file to use to get upstream DNS servers")
+	renderCmd.Flags().IPSlice("cloud-ext-lb-ips", nil, "IP Addresses of Cloud External Load Balancers for OpenShift API")
+	renderCmd.Flags().IPSlice("cloud-int-lb-ips", nil, "IP Addresses of Cloud Internal Load Balancers for OpenShift Internal API")
 	rootCmd.AddCommand(renderCmd)
 }
 
@@ -88,7 +90,15 @@ func runRender(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort)
+	apiLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips")
+	if err != nil {
+		apiLBIPs = []net.IP{}
+	}
+	apiIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips")
+	if err != nil {
+		apiIntLBIPs = []net.IP{}
+	}
+	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort, apiLBIPs, apiIntLBIPs)
 	if err != nil {
 		return err
 	}

diff --git a/pkg/config/node.go b/pkg/config/node.go
@@ -52,6 +52,10 @@ type Cluster struct {
 	VIPNetmask             int
 	MasterAmount           int64
 	NodeAddresses          []NodeAddress
+	APILBIPs               []string
+	APIIntLBIPs            []string
+	CloudLBRecordType      string
+	CloudLBEmptyType       string
 }
 
 type Backend struct {
@@ -417,37 +421,58 @@ func getNodeIpForRequestedIpStack(node v1.Node, filterIps []string, machineNetwo
 // apiPort: The port on which the k8s api listens. Should be 6443.
 // lbPort: The port on which haproxy listens.
 // statPort: The port on which the haproxy stats endpoint listens.
-func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips, ingressVips []net.IP, apiPort, lbPort, statPort uint16) (node Node, err error) {
-	vipCount := 0
-	if len(apiVips) > len(ingressVips) {
-		vipCount = len(apiVips)
-	} else {
-		vipCount = len(ingressVips)
-	}
+// apiLBIPs: A list of External Load Balancer IPs for API.
+// apiIntLBIPs: A list of Internal Load Balancer IPs for API.
+func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips, ingressVips []net.IP, apiPort, lbPort, statPort uint16, apiLBIPs, apiIntLBIPs []net.IP) (node Node, err error) {
 	nodes := []Node{}
-	var apiVip, ingressVip net.IP
-	for i := 0; i < vipCount; i++ {
-		if i < len(apiVips) {
-			apiVip = apiVips[i]
+	if len(apiIntLBIPs) == 0 {
+		// On-prem platforms
+		vipCount := 0
+		if len(apiVips) > len(ingressVips) {
+			vipCount = len(apiVips)
 		} else {
-			apiVip = nil
+			vipCount = len(ingressVips)
 		}
-		if i < len(ingressVips) {
-			ingressVip = ingressVips[i]
-		} else {
-			ingressVip = nil
+		var apiVip, ingressVip net.IP
+		for i := 0; i < vipCount; i++ {
+			if i < len(apiVips) {
+				apiVip = apiVips[i]
+			} else {
+				apiVip = nil
+			}
+			if i < len(ingressVips) {
+				ingressVip = ingressVips[i]
+			} else {
+				ingressVip = nil
+			}
+			newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, apiVip, ingressVip, apiPort, lbPort, statPort, nil, nil)
+			if err != nil {
+				return Node{}, err
+			}
+			nodes = append(nodes, newNode)
 		}
-		newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, apiVip, ingressVip, apiPort, lbPort, statPort)
-		if err != nil {
-			return Node{}, err
+	} else {
+		// Cloud Platforms with cloud LBs but no Cloud DNS
+		var apiLBIP, apiIntLBIP net.IP
+		for i := 0; i < len(apiIntLBIPs); i++ {
+			apiIntLBIP = apiIntLBIPs[i]
+			// For public clusters. Private clusters will not have External
+			// LBs so apiLBIPs could be empty.
+			if len(apiLBIPs) != 0 {
+				apiLBIP = apiLBIPs[i]
+			}
+			newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, nil, nil, 0, 0, 0, apiLBIP, apiIntLBIP)
+			if err != nil {
+				return Node{}, err
+			}
+			nodes = append(nodes, newNode)
 		}
-		nodes = append(nodes, newNode)
 	}
 	nodes[0].Configs = &nodes
 	return nodes[0], nil
 }
 
-func getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip net.IP, ingressVip net.IP, apiPort, lbPort, statPort uint16) (node Node, err error) {
+func getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip net.IP, ingressVip net.IP, apiPort, lbPort, statPort uint16, apiLBIP, apiIntLBIP net.IP) (node Node, err error) {
 	clusterName, clusterDomain, err := GetClusterNameAndDomain(kubeconfigPath, clusterConfigPath)
 	if err != nil {
 		return node, err
@@ -534,6 +559,24 @@ func getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, api
 		StatPort: statPort,
 	}
 
+	// For cloud platforms that need an in-cluster DNS solution,
+	// API and API-Int LB IPs are used to generate CoreDNS config
+	// The same DNS record type can be used for both entries for
+	// API and API-Int resolution.
+	node.Cluster.CloudLBRecordType = "A"
+	node.Cluster.CloudLBEmptyType = "AAAA"
+	if apiIntLBIP != nil {
+		node.Cluster.APIIntLBIPs = append(node.Cluster.APIIntLBIPs, apiIntLBIP.String())
+		node.Cluster.CloudLBRecordType = "A"
+		node.Cluster.CloudLBEmptyType = "AAAA"
+		if apiIntLBIP.To4() == nil {
+			node.Cluster.CloudLBRecordType = "AAAA"
+			node.Cluster.CloudLBEmptyType = "A"
+		}
+	}
+	if apiLBIP != nil {
+		node.Cluster.APILBIPs = append(node.Cluster.APILBIPs, apiLBIP.String())
+	}
 	return node, err
 }
 
@@ -694,3 +737,12 @@ func PopulateNodeAddresses(kubeconfigPath string, node *Node) {
 		}
 	}
 }
+
+func PopulateCloudLBIPAddresses(apiLBIPs, apiIntLBIPs []net.IP, node *Node) {
+	for i := 0; i < len(apiIntLBIPs); i++ {
+		node.Cluster.APIIntLBIPs = append(node.Cluster.APIIntLBIPs, apiIntLBIPs[i].String())
+	}
+	for i := 0; i < len(apiLBIPs); i++ {
+		node.Cluster.APILBIPs = append(node.Cluster.APILBIPs, apiLBIPs[i].String())
+	}
+}
diff --git a/pkg/monitor/corednsmonitor.go b/pkg/monitor/corednsmonitor.go
@@ -16,7 +16,7 @@ import (
 
 const resolvConfFilepath string = "/var/run/NetworkManager/resolv.conf"
 
-func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath string, apiVips, ingressVips []net.IP, interval time.Duration) error {
+func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath string, apiVips, ingressVips []net.IP, interval time.Duration, apiLBIPs, apiIntLBIPs []net.IP) error {
 	signals := make(chan os.Signal, 1)
 	done := make(chan bool, 1)
 
@@ -42,10 +42,14 @@ func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath strin
 			if err != nil {
 				return err
 			}
-			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, resolvConfFilepath, apiVips, ingressVips, 0, 0, 0)
+			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, resolvConfFilepath, apiVips, ingressVips, 0, 0, 0, apiLBIPs, apiIntLBIPs)
 			if err != nil {
 				return err
 			}
+			// Populate cloud LB IP addresses for platforms where the cloud LBs
+			// have already been configured
+			config.PopulateCloudLBIPAddresses(apiLBIPs, apiIntLBIPs, &newConfig)
+
 			config.PopulateNodeAddresses(kubeconfigPath, &newConfig)
 			// There should never be 0 nodes in a functioning cluster. This means
 			// we failed to populate the list, so we don't want to render.

diff --git a/pkg/monitor/dnsmasqmonitor.go b/pkg/monitor/dnsmasqmonitor.go
@@ -33,7 +33,7 @@ func DnsmasqWatch(kubeconfigPath, templatePath, cfgPath string, apiVips []net.IP
 			return nil
 		default:
 			// We only care about the api vip and cluster domain here
-			config, err := config.GetConfig(kubeconfigPath, "", "/etc/resolv.conf", apiVips, apiVips, 0, 0, 0)
+			config, err := config.GetConfig(kubeconfigPath, "", "/etc/resolv.conf", apiVips, apiVips, 0, 0, 0, []net.IP{}, []net.IP{})
 			if err != nil {
 				return err
 			}

diff --git a/pkg/monitor/dynkeepalived.go b/pkg/monitor/dynkeepalived.go
@@ -355,7 +355,7 @@ func KeepalivedWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath st
 
 		case desiredModeInfo := <-updateModeCh:
 
-			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0)
+			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0, []net.IP{}, []net.IP{})
 			if err != nil {
 				return err
 			}
@@ -437,7 +437,7 @@ func KeepalivedWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath st
 					}
 				}
 			}
-			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0)
+			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0, []net.IP{}, []net.IP{})
 			if err != nil {
 				return err
 			}