Merge pull request #168 from projectsyn/feat/enterprise-bgp-control-plane

Add support for configuring Enterprise BGP control plane

Author: simu

.cruft.json

@@ -1,13 +1,13 @@
 {
   "template": "https://github.com/projectsyn/commodore-component-template.git",
-  "commit": "98d16f99766e6c6d97322dbe42e058f0e2bf73d0",
+  "commit": "56dd2ee980f197ec2c8b3977c16458e43d0c6ef2",
   "checkout": "main",
   "context": {
     "cookiecutter": {
       "name": "Cilium",
       "slug": "cilium",
       "parameter_key": "cilium",
-      "test_cases": "defaults helm-opensource olm-opensource egress-gateway bgp-control-plane kubeproxyreplacement-strict l2-announcement clustermesh",
+      "test_cases": "defaults helm-opensource olm-opensource egress-gateway bgp-control-plane kubeproxyreplacement-strict l2-announcement clustermesh enterprise-bgp",
       "add_lib": "n",
       "add_pp": "n",
       "add_golden": "y",
@@ -25,7 +25,7 @@
       "github_name": "component-cilium",
       "github_url": "https://github.com/projectsyn/component-cilium",
       "_template": "https://github.com/projectsyn/commodore-component-template.git",
-      "_commit": "98d16f99766e6c6d97322dbe42e058f0e2bf73d0"
+      "_commit": "56dd2ee980f197ec2c8b3977c16458e43d0c6ef2"
     }
   },
   "directory": null

.github/workflows/test.yaml

@@ -40,6 +40,7 @@ jobs:
           - kubeproxyreplacement-strict
           - l2-announcement
           - clustermesh
+          - enterprise-bgp
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}
@@ -62,6 +63,7 @@ jobs:
           - kubeproxyreplacement-strict
           - l2-announcement
           - clustermesh
+          - enterprise-bgp
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}

Makefile.vars.mk

@@ -57,4 +57,4 @@ KUBENT_IMAGE    ?= ghcr.io/doitintl/kube-no-trouble:latest
 KUBENT_DOCKER   ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE)
 
 instance ?= defaults
-test_instances = tests/defaults.yml tests/helm-opensource.yml tests/olm-opensource.yml tests/egress-gateway.yml tests/bgp-control-plane.yml tests/kubeproxyreplacement-strict.yml tests/l2-announcement.yml tests/clustermesh.yml
+test_instances = tests/defaults.yml tests/helm-opensource.yml tests/olm-opensource.yml tests/egress-gateway.yml tests/bgp-control-plane.yml tests/kubeproxyreplacement-strict.yml tests/l2-announcement.yml tests/clustermesh.yml tests/enterprise-bgp.yml

class/defaults.yml

@@ -105,6 +105,7 @@ parameters:
 
     bgp:
       enabled: false
+      enterprise: false
       cluster_configs: {}
       peer_configs: {}
       auth_secrets: {}

component/bgp-control-plane.jsonnet

@@ -7,8 +7,18 @@ local util = import 'util.libsonnet';
 local inv = kap.inventory();
 local params = inv.parameters.cilium;
 
+local crd_prefix = if params.bgp.enterprise then 'Isovalent' else 'Cilium';
+local apiVersion = if params.bgp.enterprise then (
+  if util.version.minor >= 17 then
+    'isovalent.com/v1'
+  else
+    'isovalent.com/v1alpha1'
+)
+else
+  'cilium.io/v2alpha1';
+
 local CiliumBGPClusterConfig(name) =
-  kube._Object('cilium.io/v2alpha1', 'CiliumBGPClusterConfig', name) {
+  kube._Object(apiVersion, '%sBGPClusterConfig' % crd_prefix, name) {
     metadata+: {
       annotations+: {
         'argocd.argoproj.io/sync-options': 'SkipDryRunOnMissingResource=true',
@@ -18,7 +28,7 @@ local CiliumBGPClusterConfig(name) =
   };
 
 local CiliumBGPPeerConfig(name) =
-  kube._Object('cilium.io/v2alpha1', 'CiliumBGPPeerConfig', name) {
+  kube._Object(apiVersion, '%sBGPPeerConfig' % crd_prefix, name) {
     metadata+: {
       annotations+: {
         'argocd.argoproj.io/sync-options': 'SkipDryRunOnMissingResource=true',
@@ -28,7 +38,7 @@ local CiliumBGPPeerConfig(name) =
   };
 
 local CiliumBGPAdvertisement(name) =
-  kube._Object('cilium.io/v2alpha1', 'CiliumBGPAdvertisement', name) {
+  kube._Object(apiVersion, '%sBGPAdvertisement' % crd_prefix, name) {
     metadata+: {
       annotations+: {
         'argocd.argoproj.io/sync-options': 'SkipDryRunOnMissingResource=true',
@@ -38,7 +48,7 @@ local CiliumBGPAdvertisement(name) =
   };
 
 local CiliumBGPNodeConfigOverride(name) =
-  kube._Object('cilium.io/v2alpha1', 'CiliumBGPNodeConfigOverride', name) {
+  kube._Object(apiVersion, '%sBGPNodeConfigOverride' % crd_prefix, name) {
     metadata+: {
       annotations+: {
         'argocd.argoproj.io/sync-options': 'SkipDryRunOnMissingResource=true',
@@ -72,8 +82,8 @@ local render_peer_config(name, config) =
       local secretname = std.get(pconfig.spec, 'authSecretRef');
       assert
         secretname == null || std.member(auth_secret_names, secretname)
-        : "CiliumBGPPeerConfig `%s` references auth secret `%s` which doesn't exist"
-          % [ name, secretname ];
+        : "%sBGPPeerConfig `%s` references auth secret `%s` which doesn't exist"
+          % [ crd_prefix, name, secretname ];
       pconfig;
     validate_peer_config({
       metadata+: std.get(config, 'metadata', {}),
@@ -94,10 +104,10 @@ local render_cluster_config(name, config) =
       local pcfgname = std.get(pconfig, 'peerConfigRef', { name: '' }).name;
       assert
         std.member(peerConfigNames, pcfgname)
-        : 'Peer `%s` in BGP instance `%s` in CiliumBGPClusterConfig `%s` ' %
-          [ pconfig.name, iname, name ]
-          + "references CiliumBGPPeerConfig `%s` which doesn't exist" %
-            [ pcfgname ];
+        : 'Peer `%s` in BGP instance `%s` in %sBGPClusterConfig `%s` ' %
+          [ pconfig.name, iname, crd_prefix, name ]
+          + "references %sBGPPeerConfig `%s` which doesn't exist" %
+            [ crd_prefix, pcfgname ];
       pconfig;
     local render_instance(name, iconfig) =
       local cfg = iconfig {

component/olm.jsonnet

@@ -8,11 +8,22 @@ local params = inv.parameters.cilium;
 local helm = import 'render-helm-values.jsonnet';
 local util = import 'util.libsonnet';
 
+// NOTE(sg): We introduce hidden parameter `__mock_enterprise` which can be
+// set to `true` in test cases to test `params.release == enterprise` behavior
+// while downloading the opensource OLM tarball (see test case
+// `enterprise-bgp` for an example).
+// Notably, we must override this for most of the OLM processing, since we're
+// using the opensource OLM manifests for the mock enterprise test.
+local release = if std.get(params, '__mock_enterprise', false) then
+  'opensource'
+else
+  params.release;
+
 local olmDir =
   local prefix = '%s/olm/cilium/cilium-olm/' % inv.parameters._base_directory;
-  if params.release == 'opensource' then
+  if release == 'opensource' then
     prefix + 'olm-for-cilium-main/manifests/cilium.v%s/' % params.olm.full_version
-  else if params.release == 'enterprise' then
+  else if release == 'enterprise' then
     local path_variants = [
       // Known releases: 1.11.5, 1.12.6
       '',
@@ -39,7 +50,7 @@ local olmDir =
     else
       dir
   else
-    error "Unknown release '%s'" % [ params.release ];
+    error "Unknown release '%s'" % [ release ];
 
 local olmFiles = std.foldl(
   function(status, file)
@@ -98,7 +109,7 @@ local patchManifests = function(file, has_csv)
                   '--zap-log-level=%s' % params.olm.log_level,
                 ],
                 env+:
-                  if params.release == 'opensource' then
+                  if release == 'opensource' then
                     (
                       if hasK8sHost then
                         [
@@ -143,13 +154,16 @@ local patchManifests = function(file, has_csv)
         },
       } else {},
     };
+    // NOTE(sg): This is explicitly `params.relase` since we don't want the
+    // fall back to the opensource logic for the __mock_enterprise=true test
+    // cases.
     if params.release == 'enterprise' then {
       cilium+: patch,
     } else
       patch;
   if (
     file.contents.kind == 'CiliumConfig'
-    && file.contents.metadata.name == metadata_name_map[params.release].CiliumConfig
+    && file.contents.metadata.name == metadata_name_map[release].CiliumConfig
     && file.contents.metadata.namespace == 'cilium'
   ) then
     file {
@@ -158,7 +172,7 @@ local patchManifests = function(file, has_csv)
       },
     }
   else if (
-    params.release == 'enterprise'
+    release == 'enterprise'
     && file.contents.kind == 'ConfigMap'
     && file.contents.metadata.name == 'cilium-ee-olm-overrides'
     && file.contents.metadata.namespace == 'cilium'
@@ -175,7 +189,7 @@ local patchManifests = function(file, has_csv)
     }
   else if (
     file.contents.kind == 'Deployment'
-    && file.contents.metadata.name == metadata_name_map[params.release].Deployment
+    && file.contents.metadata.name == metadata_name_map[release].Deployment
     && file.contents.metadata.namespace == 'cilium'
   ) then
     file {
@@ -191,7 +205,7 @@ local patchManifests = function(file, has_csv)
           install+: {
             spec+: {
               deployments: [
-                if d.name == metadata_name_map[params.release].Deployment then
+                if d.name == metadata_name_map[release].Deployment then
                   d + deploymentPatch
                 else
                   d
@@ -216,7 +230,7 @@ local patchManifests = function(file, has_csv)
   else if (
     file.contents.kind == 'Role' &&
     file.contents.metadata.namespace == 'cilium' &&
-    file.contents.metadata.name == metadata_name_map[params.release].OlmRole
+    file.contents.metadata.name == metadata_name_map[release].OlmRole
   ) then
     file {
       contents+: {
@@ -255,7 +269,7 @@ local patchManifests = function(file, has_csv)
     }
   else if (
     file.contents.kind == 'ClusterRole' &&
-    file.contents.metadata.name == metadata_name_map[params.release].OlmClusterRole
+    file.contents.metadata.name == metadata_name_map[release].OlmClusterRole
   ) then
     file {
       contents+: {
@@ -304,7 +318,7 @@ local patchManifests = function(file, has_csv)
     file;
 
 local kubeSystemSecretRO = [
-  kube.Role(metadata_name_map[params.release].OlmRole) {
+  kube.Role(metadata_name_map[release].OlmRole) {
     metadata+: {
       namespace: 'kube-system',
     },
@@ -316,20 +330,20 @@ local kubeSystemSecretRO = [
       },
     ],
   },
-  kube.RoleBinding(metadata_name_map[params.release].OlmRole) {
+  kube.RoleBinding(metadata_name_map[release].OlmRole) {
     metadata+: {
       namespace: 'kube-system',
     },
     roleRef: {
       apiGroup: 'rbac.authorization.k8s.io',
       kind: 'Role',
-      name: metadata_name_map[params.release].OlmRole,
+      name: metadata_name_map[release].OlmRole,
     },
     subjects: [
       {
         kind: 'ServiceAccount',
         namespace: 'cilium',
-        name: metadata_name_map[params.release].OlmRole,
+        name: metadata_name_map[release].OlmRole,
       },
     ],
   },

component/render-helm-values.jsonnet

@@ -59,11 +59,33 @@ local forceBPFMasqueradeEgressGW = {
   },
 };
 
+local enterpriseBGPControlPlane =
+  if params.release != 'enterprise' then
+    std.trace('Cannot enable enterprise BGP control plane on opensource Cilium', {})
+  else if params.bgp.enterprise then
+    std.trace(
+      'User requested Enterprise BGP control plane: ' +
+      'Moving `bgpControlPlane` Helm values to `enterprise.bgpControlPlane` ' +
+      'and generating `IsovalentBGP*` resources.',
+      {
+        local bgp_values = super.bgpControlPlane,
+        bgpControlPlane: {
+          enabled: false,
+        },
+        enterprise+: {
+          bgpControlPlane+: bgp_values,
+        },
+      }
+    )
+  else
+    {};
+
 local cilium_values = std.prune(
   params.cilium_helm_values +
   replaceDeprecatedIPv4PodCIDR +
   renderPodCIDRList +
-  forceBPFMasqueradeEgressGW
+  forceBPFMasqueradeEgressGW +
+  enterpriseBGPControlPlane
 );
 
 local helm_values = {

docs/modules/ROOT/pages/references/parameters.adoc

@@ -691,6 +691,28 @@ Whether to enable the BGP control plane feature in Cilium.
 
 See the https://docs.cilium.io/en/{helm-minor-version}/network/bgp-control-plane/bgp-control-plane-v2/[upstream BGP control plane documentation] for details on the architecture and the individual custom resources mentioned in this section.
 
+=== `bgp.enterprise`
+
+[horizontal]
+type:: bool
+default:: `false`
+
+Whether to enable the enterprise BGP control plane feature in Cilium.
+This parameter has no effect if the component is used to deploy non-enterprise Cilium.
+
+When this parameter is set to `true`, the component will copy the contents of Helm value `bgpControlPlane` to Helm value `enterprise.bgpControlPlane`.
+Additionally, the component will override Helm value `bgpControlPlane.enabled` to `false`, since Cilium doesn't support having both control planes enabled simultaneously.
+This mechanism allows users to switch between the non-enterprise and enterprise BGP control plane without having to manually move Helm values.
+
+See the https://docs.isovalent.com/configuration-guide/networking/bgp/index.html[Enterprise documentation] for details on the enterprise BGP control plane.
+
+[NOTE]
+====
+The component will automatically deploy `IsovalentBGP*` resources instead of `CiliumBGP*` resources for the following parameters when the enterprise control plane is enabled.
+
+The `IsovalentBGP*` resources are structurally equivalent to the `CiliumBGP*` resources, but support the additional enterprise features, such as egress IP announcements.
+====
+
 === `bgp.cluster_configs`
 
 [horizontal]

tests/enterprise-bgp.yml

@@ -0,0 +1,53 @@
+parameters:
+  kapitan:
+    dependencies:
+      - type: https
+        source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/v6.11.3/lib/openshift4-monitoring-prom.libsonnet
+        output_path: vendor/lib/prom.libsonnet
+      - type: https
+        source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/v6.11.3/lib/openshift4-monitoring-alert-patching.libsonnet
+        output_path: vendor/lib/alert-patching.libsonnet
+
+  cilium:
+    install_method: olm
+    release: enterprise
+    __mock_enterprise: true
+    olm:
+      source:
+        enterprise: https://github.com/isovalent/olm-for-cilium/archive/main.tar.gz
+
+    bgp:
+      enabled: true
+      enterprise: true
+      cluster_configs:
+        lb-services:
+          nodeSelector:
+            matchLabels:
+              node-role.kubernetes.io/infra: ''
+          bgpInstances:
+            lbs:
+              localASN: 64512
+              peers:
+                peer1:
+                  peerAddress: 192.0.2.2
+                  peerASN: 64512
+                  peerConfigRef:
+                    name: lb-services
+                peer2:
+                  peerAddress: 192.0.2.3
+                  peerASN: 64512
+                  peerConfigRef:
+                    name: lb-services
+      peer_configs:
+        lb-services:
+          spec:
+            gracefulRestart:
+              enabled: true
+              restartTimeSeconds: 30
+          families:
+            unicast-v4:
+              afi: ipv4
+              safi: unicast
+              advertisements:
+                matchLabels:
+                  cilium.syn.tools/advertise: bgp