Adding ability to generate Corefile using LB IP addresses

A new capability is being added to cloud platforms (starting with
AWS, Azure and GCP) where the cloud LBs can be used but not the
cloud DNS. So, in-cluster DNS is provided by a CoreDNS pod during
install. The customer can optionally bring their own DNS after
install complete.

This commit adds the ability to generate a CoreDNS Corefile with
entries for API and API-Int URLs when their corresponding LB IPs
are provided. These LB IPs are not expected to change during the
life of the cluster.

cmd/corednsmonitor/corednsmonitor.go

@@ -55,8 +55,16 @@ func main() {
 			if err != nil {
 				return err
 			}
+			cloudExtLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips")
+			if err != nil {
+				cloudExtLBIPs = []net.IP{}
+			}
+			cloudIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips")
+			if err != nil {
+				cloudIntLBIPs = []net.IP{}
+			}
 
-			return monitor.CorednsWatch(args[0], clusterConfigPath, args[1], args[2], apiVips, ingressVips, checkInterval)
+			return monitor.CorednsWatch(args[0], clusterConfigPath, args[1], args[2], apiVips, ingressVips, checkInterval, cloudExtLBIPs, cloudIntLBIPs)
 		},
 	}
 	rootCmd.PersistentFlags().StringP("cluster-config", "c", "", "Path to cluster-config ConfigMap to retrieve ControlPlane info")

cmd/runtimecfg/display.go

@@ -28,6 +28,8 @@ func init() {
 	displayCmd.Flags().Uint16("lb-port", 9445, "Port where the API HAProxy LB will listen at")
 	displayCmd.Flags().Uint16("stat-port", 29445, "Port where the HAProxy stats API will listen at")
 	displayCmd.Flags().StringP("resolvconf-path", "r", "/etc/resolv.conf", "Optional path to a resolv.conf file to use to get upstream DNS servers")
+	displayCmd.Flags().IPSlice("cloud-ext-lb-ips", nil, "IP Addresses of Cloud External Load Balancers for OpenShift API")
+	displayCmd.Flags().IPSlice("cloud-int-lb-ips", nil, "IP Addresses of Cloud Internal Load Balancers for OpenShift API")
 	rootCmd.AddCommand(displayCmd)
 }
 
@@ -84,7 +86,16 @@ func runDisplay(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort)
+
+	apiLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips")
+	if err != nil {
+		apiLBIPs = []net.IP{}
+	}
+	apiIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips")
+	if err != nil {
+		apiIntLBIPs = []net.IP{}
+	}
+	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort, apiLBIPs, apiIntLBIPs)
 	if err != nil {
 		return err
 	}

cmd/runtimecfg/render.go

@@ -33,6 +33,8 @@ func init() {
 	renderCmd.Flags().Uint16("lb-port", 9445, "Port where the API HAProxy LB will listen at")
 	renderCmd.Flags().Uint16("stat-port", 29445, "Port where the HAProxy stats API will listen at")
 	renderCmd.Flags().StringP("resolvconf-path", "r", "/etc/resolv.conf", "Optional path to a resolv.conf file to use to get upstream DNS servers")
+	renderCmd.Flags().IPSlice("cloud-ext-lb-ips", nil, "IP Addresses of Cloud External Load Balancers for OpenShift API")
+	renderCmd.Flags().IPSlice("cloud-int-lb-ips", nil, "IP Addresses of Cloud Internal Load Balancers for OpenShift Internal API")
 	rootCmd.AddCommand(renderCmd)
 }
 
@@ -88,7 +90,15 @@ func runRender(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort)
+	apiLBIPs, err := cmd.Flags().GetIPSlice("cloud-ext-lb-ips")
+	if err != nil {
+		apiLBIPs = []net.IP{}
+	}
+	apiIntLBIPs, err := cmd.Flags().GetIPSlice("cloud-int-lb-ips")
+	if err != nil {
+		apiIntLBIPs = []net.IP{}
+	}
+	config, err := config.GetConfig(kubeCfgPath, clusterConfigPath, resolveConfPath, apiVips, ingressVips, apiPort, lbPort, statPort, apiLBIPs, apiIntLBIPs)
 	if err != nil {
 		return err
 	}

pkg/config/node.go

@@ -52,6 +52,10 @@ type Cluster struct {
 	VIPNetmask             int
 	MasterAmount           int64
 	NodeAddresses          []NodeAddress
+	APILBIP                string
+	APIIntLBIP             string
+	CloudLBRecordType      string
+	CloudLBEmptyType       string
 }
 
 type Backend struct {
@@ -417,7 +421,9 @@ func getNodeIpForRequestedIpStack(node v1.Node, filterIps []string, machineNetwo
 // apiPort: The port on which the k8s api listens. Should be 6443.
 // lbPort: The port on which haproxy listens.
 // statPort: The port on which the haproxy stats endpoint listens.
-func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips, ingressVips []net.IP, apiPort, lbPort, statPort uint16) (node Node, err error) {
+// apiLBIPs: A list of External Load Balancer IPs for API.
+// apiIntLBIPs: A list of Internal Load Balancer IPs for API.
+func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips, ingressVips []net.IP, apiPort, lbPort, statPort uint16, apiLBIPs, apiIntLBIPs []net.IP) (node Node, err error) {
 	vipCount := 0
 	if len(apiVips) > len(ingressVips) {
 		vipCount = len(apiVips)
@@ -437,7 +443,7 @@ func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips
 		} else {
 			ingressVip = nil
 		}
-		newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, apiVip, ingressVip, apiPort, lbPort, statPort)
+		newNode, err := getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath, apiVip, ingressVip, apiPort, lbPort, statPort, apiLBIPs[0], apiIntLBIPs[0])
 		if err != nil {
 			return Node{}, err
 		}
@@ -447,7 +453,7 @@ func GetConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVips
 	return nodes[0], nil
 }
 
-func getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip net.IP, ingressVip net.IP, apiPort, lbPort, statPort uint16) (node Node, err error) {
+func getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, apiVip net.IP, ingressVip net.IP, apiPort, lbPort, statPort uint16, apiLBIP, apiIntLBIP net.IP) (node Node, err error) {
 	clusterName, clusterDomain, err := GetClusterNameAndDomain(kubeconfigPath, clusterConfigPath)
 	if err != nil {
 		return node, err
@@ -534,6 +540,22 @@ func getNodeConfig(kubeconfigPath, clusterConfigPath, resolvConfPath string, api
 		StatPort: statPort,
 	}
 
+	// For cloud platforms that need an in-cluster DNS solution,
+	// API and API-Int LB IPs are used to generate CoreDNS config
+	// The same DNS record type can be used for both entries for
+	// API and API-Int resolution.
+	node.Cluster.CloudLBRecordType = "A"
+	node.Cluster.CloudLBEmptyType = "AAAA"
+	if apiIntLBIP != nil {
+		node.Cluster.APILBIP = apiIntLBIP.String()
+		if apiIntLBIP.To4() == nil {
+			node.Cluster.CloudLBRecordType = "AAAA"
+			node.Cluster.CloudLBEmptyType = "A"
+		}
+	}
+	if apiLBIP != nil {
+		node.Cluster.APILBIP = apiLBIP.String()
+	}
 	return node, err
 }
 

pkg/monitor/corednsmonitor.go

@@ -16,7 +16,7 @@ import (
 
 const resolvConfFilepath string = "/var/run/NetworkManager/resolv.conf"
 
-func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath string, apiVips, ingressVips []net.IP, interval time.Duration) error {
+func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath string, apiVips, ingressVips []net.IP, interval time.Duration, apiLBIPs, apiIntLBIPs []net.IP) error {
 	signals := make(chan os.Signal, 1)
 	done := make(chan bool, 1)
 
@@ -42,7 +42,7 @@ func CorednsWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath strin
 			if err != nil {
 				return err
 			}
-			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, resolvConfFilepath, apiVips, ingressVips, 0, 0, 0)
+			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, resolvConfFilepath, apiVips, ingressVips, 0, 0, 0, apiLBIPs, apiIntLBIPs)
 			if err != nil {
 				return err
 			}

pkg/monitor/dnsmasqmonitor.go

@@ -33,7 +33,7 @@ func DnsmasqWatch(kubeconfigPath, templatePath, cfgPath string, apiVips []net.IP
 			return nil
 		default:
 			// We only care about the api vip and cluster domain here
-			config, err := config.GetConfig(kubeconfigPath, "", "/etc/resolv.conf", apiVips, apiVips, 0, 0, 0)
+			config, err := config.GetConfig(kubeconfigPath, "", "/etc/resolv.conf", apiVips, apiVips, 0, 0, 0, []net.IP{}, []net.IP{})
 			if err != nil {
 				return err
 			}

pkg/monitor/dynkeepalived.go

@@ -355,7 +355,7 @@ func KeepalivedWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath st
 
 		case desiredModeInfo := <-updateModeCh:
 
-			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0)
+			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0, []net.IP{}, []net.IP{})
 			if err != nil {
 				return err
 			}
@@ -437,7 +437,7 @@ func KeepalivedWatch(kubeconfigPath, clusterConfigPath, templatePath, cfgPath st
 					}
 				}
 			}
-			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0)
+			newConfig, err := config.GetConfig(kubeconfigPath, clusterConfigPath, "/etc/resolv.conf", apiVips, ingressVips, 0, 0, 0, []net.IP{}, []net.IP{})
 			if err != nil {
 				return err
 			}