feat: support nacos storage

.github/workflows/build-and-test.yaml

@@ -134,7 +134,41 @@ jobs:
 
     - name: "Run Higress E2E Conformance Tests"
       run: GOPROXY="https://proxy.golang.org,direct" make higress-conformance-test
-
+
+  higress-conformance-test-nacos:
+    runs-on: ubuntu-latest
+    needs: [ build ]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+
+      - name: Setup Golang Caches
+        uses: actions/cache@v3
+        with:
+          path: |-
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ github.run_id }}
+          restore-keys: |
+            ${{ runner.os }}-go
+
+      - name: Setup Submodule Caches
+        uses: actions/cache@v3
+        with:
+          path: |-
+            .git/modules
+          key: ${{ runner.os }}-submodules-cache-${{ github.run_id }}
+          restore-keys: ${{ runner.os }}-submodules-cache
+
+      - run: git stash # restore patch
+
+      - name: "Run Higress E2E Conformance Nacos Tests"
+        run: GOPROXY="https://proxy.golang.org,direct" make higress-conformance-test-nacos
+
   publish:
     runs-on: ubuntu-latest
     needs: [higress-conformance-test,gateway-conformance-test]

Makefile.core.mk

@@ -245,6 +245,14 @@ higress-conformance-test: $(tools/kind) delete-cluster create-cluster docker-bui
 .PHONY: higress-wasmplugin-test
 higress-wasmplugin-test: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-wasmplugin run-higress-e2e-test-wasmplugin delete-cluster
 
+# higress-conformance-test-nacos runs ingress nacos tests.
+.PHONY: higress-conformance-test-nacos
+higress-conformance-test-nacos: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-nacos port-forward run-higress-e2e-test-nacos delete-cluster
+
+.PHONY: port-forward
+port-forward:
+	kubectl port-forward -n higress-system svc/nacos-service 8848:8848 &
+
 # create-cluster creates a kube cluster with kind.
 .PHONY: create-cluster
 create-cluster: $(tools/kind)
@@ -293,3 +301,12 @@ run-higress-e2e-test-wasmplugin:
 	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
 	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
 	go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true
+
+.PHONY: run-higress-e2e-test-nacos
+run-higress-e2e-test-nacos:
+	@echo -e "\n\033[36mRunning higress conformance tests...\033[0m"
+	@echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available
+	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
+	go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --debug=true --enableApiServer=true

helm/core/templates/configmap.yaml

@@ -105,28 +105,7 @@ metadata:
   labels:
     {{- include "gateway.labels" . | nindent 4 }}    
 data:
-  {{- if .Values.apiserver.enabled }}
-  kubeconfig: |
-    apiVersion: v1
-    kind: Config
-    clusters:
-      - name: {{ .Values.clusterName | default "higress" }}
-        cluster:
-          server: https://{{ .Values.apiserver.addr }}:{{ .Values.apiserver.securePort }}
-          insecure-skip-tls-verify: true
-    users:
-      - name: higress-admin
-        user:
-          client-certificate-data: {{ .Files.Get "files/api/client.crt" | b64enc }}
-          client-key-data: {{ .Files.Get "files/api/client.key" | b64enc }}
-    contexts:
-      - name: higress
-        context:
-          cluster: higress
-          user: higress-admin
-    preferences: {}
-    current-context: higress
-  {{- end }}
+
   # Configuration file for the mesh networks to be used by the Split Horizon EDS.
   meshNetworks: |-
   {{- if .Values.global.meshNetworks }}

test/e2e/conformance/utils/configcenter/nacos/nacos.go

@@ -0,0 +1,97 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package nacos
+
+import (
+	"net/url"
+	"strconv"
+	"strings"
+
+	cc "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter"
+	"github.com/nacos-group/nacos-sdk-go/v2/clients"
+	"github.com/nacos-group/nacos-sdk-go/v2/clients/config_client"
+	"github.com/nacos-group/nacos-sdk-go/v2/common/constant"
+	"github.com/nacos-group/nacos-sdk-go/v2/vo"
+)
+
+type storage struct {
+	client config_client.IConfigClient
+}
+
+func NewClient(addr string) (cc.Storage, error) {
+	clientConfig := constant.NewClientConfig(
+		constant.WithNamespaceId(""),
+		constant.WithUsername(""),
+		constant.WithPassword(""),
+		constant.WithLogLevel("info"),
+	)
+
+	serverUrl, err := url.Parse(addr)
+	if err != nil {
+		return nil, err
+	}
+	rawPort := serverUrl.Port()
+	var port uint64
+	if rawPort != "" {
+		port, err = strconv.ParseUint(rawPort, 10, 0)
+		if err != nil || port < 1 || port > 65535 {
+			return nil, err
+		}
+	} else {
+		port = 80
+	}
+	path := serverUrl.Path
+	if strings.HasSuffix(path, "/") {
+		path = path[:len(path)-1]
+	}
+	serverConfigs := []constant.ServerConfig{
+		{
+			IpAddr:      serverUrl.Hostname(),
+			ContextPath: path,
+			Port:        port,
+			Scheme:      serverUrl.Scheme,
+		},
+	}
+
+	client, err := clients.NewConfigClient(
+		vo.NacosClientParam{
+			ClientConfig:  clientConfig,
+			ServerConfigs: serverConfigs,
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+	return storage{
+		client: client,
+	}, nil
+}
+
+func (s storage) PublishConfig(dataId, group, content string) error {
+	_, err := s.client.PublishConfig(vo.ConfigParam{
+		DataId:  dataId,
+		Group:   group,
+		Content: content,
+	})
+	return err
+}
+
+func (s storage) DeleteConfig(dataId, group string) error {
+	_, err := s.client.DeleteConfig(vo.ConfigParam{
+		DataId: dataId,
+		Group:  group,
+	})
+	return err
+}

test/e2e/conformance/utils/configcenter/storage.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cc
+
+type Storage interface {
+	PublishConfig(dataId, group, content string) error
+	DeleteConfig(dataId, group string) error
+}

test/e2e/conformance/utils/flags/flags.go

@@ -27,4 +27,7 @@ var (
 	WasmPluginType       = flag.String("wasmPluginType", "GO", "Define wasm plugin type, currently supports GO, CPP")
 	WasmPluginName       = flag.String("wasmPluginName", "", "Define wasm plugin name")
 	IsEnvoyConfigTest    = flag.Bool("isEnvoyConfigTest", false, "Determine if run envoy config conformance test")
+	EnableApiServer      = flag.Bool("enableApiServer", false, "Determine if enable api server")
+	// TODO when apiserver support other mode, we can use string field like: nacos://username:password@ip:port/namespace
+	Storage = flag.String("storage", "http://127.0.0.1:8848", "Define storage")
 )

test/e2e/conformance/utils/kubernetes/apply.go

@@ -24,6 +24,7 @@ import (
 	"testing"
 
 	ingress "github.com/alibaba/higress/test/e2e/conformance"
+	cc "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter"
 	"github.com/stretchr/testify/require"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -249,3 +250,37 @@ func getContentsFromPathOrURL(location string, timeoutConfig config.TimeoutConfi
 	}
 	return bytes.NewBuffer(b), nil
 }
+
+// MustPublishConfig publish config to config center
+func (a Applier) MustPublishConfig(t *testing.T, timeoutConfig config.TimeoutConfig, location string, cleanup bool, cc cc.Storage) {
+	data, err := getContentsFromPathOrURL(location, timeoutConfig)
+	require.NoError(t, err)
+
+	decoder := yaml.NewYAMLOrJSONDecoder(data, 4096)
+
+	resources, err := a.prepareResources(t, decoder)
+	if err != nil {
+		t.Logf("🧳 Manifest: %s", data.String())
+		require.NoErrorf(t, err, "error parsing manifest")
+	}
+
+	for _, r := range resources {
+		// TODO: Maybe add s or es
+		dataId := r.GetKind() + "es"
+		group := r.GetNamespace()
+		// convert r to string
+		var content []byte
+		content, err = r.MarshalJSON()
+		require.NoError(t, err)
+		// publish
+		err = cc.PublishConfig(dataId, group, string(content))
+		require.NoError(t, err)
+		if cleanup {
+			t.Cleanup(func() {
+				// delete
+				err = cc.DeleteConfig(dataId, group)
+				require.NoError(t, err)
+			})
+		}
+	}
+}

test/e2e/conformance/utils/suite/suite.go

@@ -15,11 +15,14 @@ package suite
 
 import (
 	"fmt"
+	"github.com/alibaba/higress/test/e2e/conformance/utils/configcenter/nacos"
 	"testing"
 
 	"github.com/alibaba/higress/test/e2e/conformance/utils/config"
+	cc "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter"
 	"github.com/alibaba/higress/test/e2e/conformance/utils/kubernetes"
 	"github.com/alibaba/higress/test/e2e/conformance/utils/roundtripper"
+	"github.com/stretchr/testify/require"
 	"istio.io/istio/pilot/pkg/util/sets"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -38,6 +41,9 @@ type ConformanceTestSuite struct {
 	SkipTests         sets.Set
 	TimeoutConfig     config.TimeoutConfig
 	SupportedFeatures sets.Set
+	EnableApiServer   bool
+	Storage           string
+	ConfigCenter      cc.Storage
 }
 
 // Options can be used to initialize a ConformanceTestSuite.
@@ -63,6 +69,9 @@ type Options struct {
 
 	// IsEnvoyConfigTest indicates whether or not the test is for envoy config
 	IsEnvoyConfigTest bool
+
+	EnableApiServer bool
+	Storage         string
 }
 
 type WASMOptions struct {
@@ -112,7 +121,9 @@ func New(s Options) *ConformanceTestSuite {
 		Applier: kubernetes.Applier{
 			NamespaceLabels: s.NamespaceLabels,
 		},
-		TimeoutConfig: s.TimeoutConfig,
+		TimeoutConfig:   s.TimeoutConfig,
+		EnableApiServer: s.EnableApiServer,
+		Storage:         s.Storage,
 	}
 
 	// apply defaults
@@ -132,6 +143,14 @@ func New(s Options) *ConformanceTestSuite {
 // Setup ensures the base resources required for conformance tests are installed
 // in the cluster. It also ensures that all relevant resources are ready.
 func (suite *ConformanceTestSuite) Setup(t *testing.T) {
+	if suite.EnableApiServer {
+		t.Logf("📦 Test Setup: Ensuring ApiServer Storage has been accepted")
+		configClient, err := nacos.NewClient(suite.Storage)
+		require.NoError(t, err)
+		require.NotNil(t, configClient)
+		suite.ConfigCenter = configClient
+	}
+
 	t.Logf("📦 Test Setup: Ensuring IngressClass has been accepted")
 
 	suite.Applier.IngressClass = suite.IngressClassName
@@ -220,9 +239,16 @@ func (test *ConformanceTest) Run(t *testing.T, suite *ConformanceTestSuite) {
 		suite.Applier.MustDelete(t, suite.Client, suite.TimeoutConfig, manifestLocation)
 	}
 
-	for _, manifestLocation := range test.Manifests {
-		t.Logf("🧳 Applying Manifests: %s", manifestLocation)
-		suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, !test.NotCleanup)
+	if suite.EnableApiServer {
+		for _, manifestLocation := range test.Manifests {
+			t.Logf("🧳 Applying ApiServer Storage Manifests")
+			suite.Applier.MustPublishConfig(t, suite.TimeoutConfig, manifestLocation, !test.NotCleanup, suite.ConfigCenter)
+		}
+	} else {
+		for _, manifestLocation := range test.Manifests {
+			t.Logf("🧳 Applying Manifests: %s", manifestLocation)
+			suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, !test.NotCleanup)
+		}
 	}
 
 	test.Test(t, suite)

test/e2e/e2e_test.go

@@ -51,6 +51,8 @@ func TestHigressConformanceTests(t *testing.T) {
 		},
 		GatewayAddress:             "localhost",
 		EnableAllSupportedFeatures: true,
+		EnableApiServer:            *flags.EnableApiServer,
+		Storage:                    *flags.Storage,
 		IsEnvoyConfigTest:          *flags.IsEnvoyConfigTest,
 	})