diff --git a/.github/workflows/build-and-test-plugin.yaml b/.github/workflows/build-and-test-plugin.yaml index 363bd4ae9d..65bfef12ea 100644 --- a/.github/workflows/build-and-test-plugin.yaml +++ b/.github/workflows/build-and-test-plugin.yaml @@ -76,6 +76,53 @@ jobs: retry_on: error command: GOPROXY="https://proxy.golang.org,direct" PLUGIN_TYPE=${{ matrix.wasmPluginType }} make higress-wasmplugin-test + higress-wasmplugin-test-nacos: + runs-on: ubuntu-latest + strategy: + matrix: + wasmPluginType: [ GO ] + wasmPluginName: [ request-block ] + steps: + - uses: actions/checkout@v3 + + - name: Free Up GitHub Actions Ubuntu Runner Disk Space 🔧 + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + + - name: "Setup Go" + uses: actions/setup-go@v3 + with: + go-version: 1.19 + + - name: Setup Golang Caches + uses: actions/cache@v3 + with: + path: |- + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-go + + - name: Setup Submodule Caches + uses: actions/cache@v3 + with: + path: |- + .git/modules + key: ${{ runner.os }}-submodules-cache-${{ github.run_id }} + restore-keys: ${{ runner.os }}-submodules-cache + + - run: git stash # restore patch + + - name: "Run Ingress WasmPlugins Tests" + run: GOPROXY="https://proxy.golang.org,direct" PLUGIN_TYPE=${{ matrix.wasmPluginType }} PLUGIN_NAME=${{ matrix.wasmPluginName }} make higress-wasmplugin-test + publish: runs-on: ubuntu-latest needs: [ higress-wasmplugin-test ] diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 44e42b7c2b..fa52b7a080 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -106,6 +106,16 @@ jobs: needs: [build] steps: - uses: actions/checkout@v3 + + - name: Free Up GitHub Actions Ubuntu Runner Disk Space 🔧 + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true - name: "Setup Go" uses: actions/setup-go@v3 @@ -134,7 +144,51 @@ jobs: - name: "Run Higress E2E Conformance Tests" run: GOPROXY="https://proxy.golang.org,direct" make higress-conformance-test - + + higress-conformance-test-nacos: + runs-on: ubuntu-latest + needs: [ build ] + steps: + - uses: actions/checkout@v3 + + - name: Free Up GitHub Actions Ubuntu Runner Disk Space 🔧 + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true + + - name: "Setup Go" + uses: actions/setup-go@v3 + with: + go-version: 1.19 + + - name: Setup Golang Caches + uses: actions/cache@v3 + with: + path: |- + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-go + + - name: Setup Submodule Caches + uses: actions/cache@v3 + with: + path: |- + .git/modules + key: ${{ runner.os }}-submodules-cache-${{ github.run_id }} + restore-keys: ${{ runner.os }}-submodules-cache + + - run: git stash # restore patch + + - name: "Run Higress E2E Conformance Nacos Tests" + run: GOPROXY="https://proxy.golang.org,direct" make higress-conformance-test-nacos + publish: runs-on: ubuntu-latest needs: [higress-conformance-test,gateway-conformance-test] diff --git a/Makefile.core.mk b/Makefile.core.mk index 3d2d89d984..dcba07cdc8 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -184,6 +184,12 @@ install-dev: pre-install helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' install-dev-wasmplugin: build-wasmplugins pre-install helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'global.volumeWasmPlugins=true' --set 'global.onlyPushRouteCluster=false' +install-dev-nacos: pre-install + tools/hack/gen-secret-configmap.sh + helm install higress helm/core -n higress-system --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'nacos.enabled=true' +install-dev-wasmplugin-nacos: build-wasmplugins pre-install + tools/hack/gen-secret-configmap.sh + helm install higress helm/core -n higress-system --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'apiserver.enabled=true' --set 'nacos.enabled=true' --set 'global.volumeWasmPlugins=true' --set 'global.onlyPushRouteCluster=false' uninstall: helm uninstall higress -n higress-system @@ -257,6 +263,19 @@ higress-wasmplugin-test: $(tools/kind) delete-cluster create-cluster docker-buil .PHONY: higress-wasmplugin-test-clean higress-wasmplugin-test-clean: $(tools/kind) delete-cluster +# higress-conformance-test-nacos runs ingress nacos tests. +.PHONY: higress-conformance-test-nacos +higress-conformance-test-nacos: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-nacos port-forward-nacos run-higress-e2e-test-nacos delete-cluster + +# higress-wasmplugin-test-nacos runs ingress wasmplugin nacos tests. +.PHONY: higress-wasmplugin-test-nacos +higress-wasmplugin-test-nacos: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-wasmplugin-nacos port-forward-nacos run-higress-e2e-test-wasmplugin-nacos delete-cluster + +.PHONY: port-forward-nacos +port-forward-nacos: + kubectl wait --timeout=10m -n higress-system --for=condition=Ready pod -l app.kubernetes.io/name=higress-nacos,app.kubernetes.io/instance=higress + kubectl port-forward -n higress-system svc/higress-nacos-service 8848:8848 9848:9848 & + # create-cluster creates a kube cluster with kind. .PHONY: create-cluster create-cluster: $(tools/kind) @@ -276,12 +295,14 @@ kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster us tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG) tools/hack/docker-pull-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 tools/hack/docker-pull-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 + tools/hack/docker-pull-image.sh docker.io/nacos/nacos-server latest tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0 tools/hack/docker-pull-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 tools/hack/docker-pull-image.sh docker.io/bitinit/eureka latest tools/hack/docker-pull-image.sh docker.io/alihigress/httpbin 1.0.2 tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/dubbo-provider-demo 0.0.3-x86 tools/hack/kind-load-image.sh docker.io/alihigress/nacos-standlone-rc3 1.0.0-RC3 + tools/hack/kind-load-image.sh docker.io/nacos/nacos-server latest tools/hack/kind-load-image.sh docker.io/hashicorp/consul 1.16.0 tools/hack/kind-load-image.sh docker.io/alihigress/httpbin 1.0.2 tools/hack/kind-load-image.sh docker.io/charlie1380/eureka-registry-provider v0.3.0 @@ -366,3 +387,22 @@ run-higress-e2e-test-wasmplugin-clean: @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --debug=true --test-area=clean + +.PHONY: run-higress-e2e-test-nacos +run-higress-e2e-test-nacos: + @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" + @echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n" + kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available + @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" + kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available + go test -v -tags conformance ./test/e2e/e2e_test.go --ingress-class=higress --enableApiServer=true + +.PHONY: run-higress-e2e-test-wasmplugin-nacos +run-higress-e2e-test-wasmplugin-nacos: + @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" + @echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n" + kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available + @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" + kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available + go test -v -tags conformance ./test/e2e/e2e_test.go -isWasmPluginTest=true -wasmPluginType=$(PLUGIN_TYPE) -wasmPluginName=$(PLUGIN_NAME) --ingress-class=higress --enableApiServer=true + diff --git a/docker/Dockerfile.higress b/docker/Dockerfile.higress index cd52b74ee5..3527dd8884 100644 --- a/docker/Dockerfile.higress +++ b/docker/Dockerfile.higress @@ -13,7 +13,11 @@ FROM ${HUB}/base:${BASE_VERSION} ARG TARGETARCH COPY ${TARGETARCH:-amd64}/higress /usr/local/bin/higress +COPY wait-for-apiserver.sh /usr/local/bin/wait-for-apiserver.sh +COPY run-higress.sh /usr/local/bin/run-higress.sh + +RUN chmod +x /usr/local/bin/*.sh USER 1337:1337 -ENTRYPOINT ["/usr/local/bin/higress"] +ENTRYPOINT ["/usr/local/bin/run-higress.sh"] diff --git a/docker/docker.mk b/docker/docker.mk index aa01abdfc0..d7fa7a762c 100644 --- a/docker/docker.mk +++ b/docker/docker.mk @@ -14,12 +14,16 @@ docker.higress: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB} docker.higress: $(OUT_LINUX)/higress +docker.higress: docker/wait-for-apiserver.sh +docker.higress: docker/run-higress.sh docker.higress: docker/Dockerfile.higress $(HIGRESS_DOCKER_RULE) docker.higress-buildx: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB} docker.higress-buildx: $(AMD64_OUT_LINUX)/higress docker.higress-buildx: $(ARM64_OUT_LINUX)/higress +docker.higress: docker/wait-for-apiserver.sh +docker.higress: docker/run-higress.sh docker.higress-buildx: docker/Dockerfile.higress $(HIGRESS_DOCKER_BUILDX_RULE) diff --git a/docker/run-higress.sh b/docker/run-higress.sh new file mode 100644 index 0000000000..7444c4d961 --- /dev/null +++ b/docker/run-higress.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright (c) 2022 Alibaba Group Holding Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http:www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +args=("$@") + +for arg in "${args[@]}"; do + echo "Argument: $arg" +done + +/usr/local/bin/wait-for-apiserver.sh + +echo "starting higress..." + +exec /usr/local/bin/higress "$@" \ No newline at end of file diff --git a/docker/wait-for-apiserver.sh b/docker/wait-for-apiserver.sh new file mode 100644 index 0000000000..d08ca4e84c --- /dev/null +++ b/docker/wait-for-apiserver.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright (c) 2022 Alibaba Group Holding Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http:www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# if has HIGRESS_APISERVER_SVC env, wait for apiserver ready +if [ -n "$HIGRESS_APISERVER_SVC" ]; then + while true; do + echo "testing higress apiserver is ready to connect..." + nc -z "$HIGRESS_APISERVER_SVC" "${HIGRESS_APISERVER_PORT}" + if [ $? -eq 0 ]; then + break + fi + sleep 1 + done +fi \ No newline at end of file diff --git a/helm/core/templates/_helpers.tpl b/helm/core/templates/_helpers.tpl index 392fd04ef4..103dfe7089 100644 --- a/helm/core/templates/_helpers.tpl +++ b/helm/core/templates/_helpers.tpl @@ -101,3 +101,7 @@ higress: {{ include "controller.name" . }} true {{- end }} {{- end }} + +{{- define "apiserver.name" -}} +{{- .Values.apiserver.name | default "higress-apiserver" -}} +{{- end }} \ No newline at end of file diff --git a/helm/core/templates/controller-deployment.yaml b/helm/core/templates/controller-deployment.yaml index 0c37811f7a..b2a1622b4e 100644 --- a/helm/core/templates/controller-deployment.yaml +++ b/helm/core/templates/controller-deployment.yaml @@ -195,6 +195,9 @@ spec: image: "{{ .Values.controller.hub | default .Values.global.hub }}/{{ .Values.controller.image | default "higress" }}:{{ .Values.controller.tag | default .Chart.AppVersion }}" args: - "serve" + {{- if .Values.apiserver.enabled }} + - --kubeconfig=/etc/istio/config/kubeconfig + {{- end }} - --gatewaySelectorKey=higress - --gatewaySelectorValue={{ .Release.Namespace }}-{{ include "gateway.name" . }} - --gatewayHttpPort={{ .Values.gateway.httpPort }} @@ -207,6 +210,12 @@ spec: - --watchNamespace={{ .Values.global.watchNamespace }} {{- end }} env: + {{- if .Values.apiserver.enabled }} + - name: HIGRESS_APISERVER_SVC + value: {{ .Values.apiserver.addr | quote }} + - name: HIGRESS_APISERVER_PORT + value: {{ .Values.apiserver.securePort | quote }} + {{- end }} - name: POD_NAME valueFrom: fieldRef: @@ -245,6 +254,96 @@ spec: volumeMounts: - name: log mountPath: /var/log + {{- if .Values.apiserver.enabled }} + - name: config + mountPath: /etc/istio/config + {{- end }} + {{- if .Values.apiserver.enabled }} + - name: {{ .Values.apiserver.name }} + image: "{{ .Values.apiserver.hub | default .Values.global.hub }}/{{ .Values.apiserver.image }}:{{ .Values.apiserver.tag }}" + imagePullPolicy: {{ .Values.apiserver.imagePullPolicy }} + args: + - --secure-port + - {{ .Values.apiserver.securePort | quote }} + - --client-ca-file + - /etc/api/ca.crt + - --tls-cert-file + - /etc/api/server.crt + - --tls-private-key-file + - /etc/api/server.key + - --storage + - {{ .Values.apiserver.storage }} + - --{{ .Values.apiserver.storage }}-server + - {{ .Values.apiserver.serverAddr }} + - --{{ .Values.apiserver.storage }}-username + - {{ .Values.apiserver.username }} + - --{{ .Values.apiserver.storage }}-password + - {{ .Values.apiserver.password }} + - --{{ .Values.apiserver.storage }}-ns-id + - {{ .Values.apiserver.namespaceID }} + - --{{ .Values.apiserver.storage }}-encryption-key-file + - /etc/api/nacos.key + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: INSTANCE_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: HOST_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + ports: + - containerPort: 8443 + hostPort: 8443 + name: https + protocol: TCP + readinessProbe: + failureThreshold: 30 + httpGet: + path: /readyz + port: 8443 + scheme: HTTPS + initialDelaySeconds: 1 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 3 + resources: { } + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 1337 + runAsNonRoot: true + runAsUser: 1337 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/api + name: cert-config + readOnly: true + - mountPath: /tmp/nacos + name: nacos-data + {{- end }} {{- with .Values.controller.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -260,10 +359,28 @@ spec: volumes: - name: log emptyDir: {} + {{- if .Values.apiserver.enabled }} + - name: {{ .Values.apiserver.storage }}-data + emptyDir: {} + - name: cert-config + secret: + defaultMode: 420 + secretName: higress-apiserver + {{- end }} {{- if not .Values.global.enableHigressIstio }} - name: config + {{- if not .Values.apiserver.enabled }} configMap: name: higress-config + {{- end }} + {{- if .Values.apiserver.enabled }} + projected: + sources: + - configMap: + name: higress-config + - configMap: + name: higress-apiserver + {{- end }} # Technically not needed on this pod - but it helps debugging/testing SDS # Should be removed after everything works. - emptyDir: diff --git a/helm/core/templates/controller-service.yaml b/helm/core/templates/controller-service.yaml index 70b22351e4..938d7cf1f3 100644 --- a/helm/core/templates/controller-service.yaml +++ b/helm/core/templates/controller-service.yaml @@ -23,6 +23,12 @@ spec: - port: 15014 name: http-monitoring # prometheus stats protocol: TCP + {{- if .Values.apiserver.enabled }} + - port: {{ .Values.apiserver.securePort }} + name: https-apiserver + protocol: TCP + targetPort: {{ .Values.apiserver.securePort }} + {{- end }} {{- end }} selector: - {{- include "controller.selectorLabels" . | nindent 4 }} + {{- include "controller.selectorLabels" . | nindent 4 }} \ No newline at end of file diff --git a/helm/core/templates/nacos-service.yaml b/helm/core/templates/nacos-service.yaml new file mode 100644 index 0000000000..d2c1a43b8b --- /dev/null +++ b/helm/core/templates/nacos-service.yaml @@ -0,0 +1,30 @@ +{{- if .Values.nacos.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.nacos.name }}-service + namespace: {{ .Release.Namespace }} +spec: + type: {{ .Values.nacos.service.type }} + ports: + - port: {{ .Values.nacos.service.port }} + targetPort: {{ .Values.nacos.service.port }} + protocol: TCP + name: http + - port: {{ add .Values.nacos.service.port 1000}} + name: client-rpc + targetPort: {{add .Values.nacos.service.port 1000}} + - port: {{add .Values.nacos.service.port 1001}} + name: raft-rpc + targetPort: {{add .Values.nacos.service.port 1001}} + - port: 7848 + name: old-raft-rpc + targetPort: 7848 + protocol: TCP + {{- if eq .Values.nacos.service.type "NodePort" }} + nodePort: {{ .Values.nacos.service.nodePort }} + {{- end }} + selector: + app.kubernetes.io/name: {{ .Values.nacos.name }} + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end}} \ No newline at end of file diff --git a/helm/core/templates/nacos-statefulset.yaml b/helm/core/templates/nacos-statefulset.yaml new file mode 100644 index 0000000000..3d76e0de50 --- /dev/null +++ b/helm/core/templates/nacos-statefulset.yaml @@ -0,0 +1,104 @@ +{{- if .Values.nacos.enabled }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Values.nacos.name }} + namespace: {{ .Release.Namespace }} + annotations: + {{- toYaml .Values.nacos.annotations | indent 4 }} +spec: + serviceName: {{ .Values.nacos.name }}-service + replicas: {{ .Values.nacos.replica }} + {{- if .Values.nacos.podManagementPolicy }} + podManagementPolicy: {{ .Values.nacos.podManagementPolicy }} + {{- else}} + podManagementPolicy: OrderedReady + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: {{ .Values.nacos.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ .Values.nacos.name }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + {{- with .Values.nacos.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nacos.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nacos.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ .Values.nacos.name }} + image: "{{ .Values.nacos.hub }}:{{ .Values.nacos.tag }}" + imagePullPolicy: {{ .Values.nacos.pullPolicy }} + startupProbe: + initialDelaySeconds: 180 + periodSeconds: 5 + timeoutSeconds: 10 + httpGet: + scheme: HTTP + port: {{ .Values.nacos.service.port }} + path: /nacos/v1/console/health/readiness + livenessProbe: + initialDelaySeconds: 10 + periodSeconds: 5 + timeoutSeconds: 10 + httpGet: + scheme: HTTP + port: {{ .Values.nacos.service.port }} + path: /nacos/v1/console/health/liveness + ports: + - name: http + containerPort: {{ .Values.nacos.service.port }} + protocol: TCP + - containerPort: {{ add .Values.nacos.service.port 1000}} + name: client-rpc + - containerPort: {{ add .Values.nacos.service.port 1001 }} + name: raft-rpc + - containerPort: 7848 + name: old-raft-rpc + resources: + {{- toYaml .Values.nacos.resources | nindent 12 }} + env: + - name: NACOS_SERVER_PORT + value: {{ .Values.nacos.service.port | quote }} + - name: NACOS_APPLICATION_PORT + value: {{ .Values.nacos.service.port | quote }} + - name: PREFER_HOST_MODE + value: {{ .Values.nacos.preferHostMode | quote }} + - name: MODE + value: "standalone" + - name: EMBEDDED_STORAGE + value: embedded + volumeMounts: + - name: data + mountPath: /home/nacos/plugins/peer-finder + subPath: peer-finder + - name: data + mountPath: /home/nacos/data + subPath: data + - name: data + mountPath: /home/nacos/logs + subPath: logs + {{- if not .Values.nacos.persistence.enabled }} + volumes: + - name: data + emptyDir: { } + {{- end }} + {{- if .Values.nacos.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: data + spec: + {{- toYaml .Values.nacos.persistence.data | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/core/values.yaml b/helm/core/values.yaml index 9ce784e9a8..1b37a7180c 100644 --- a/helm/core/values.yaml +++ b/helm/core/values.yaml @@ -639,6 +639,23 @@ pilot: # Additional labels to apply on the pod level for monitoring and logging configuration. podLabels: {} +## Higress ApiServer Storage Settings +## Todo simplify this +apiserver: + addr: 127.0.0.1 + enabled: false + name: "higress-apiserver" + hub: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress + tag: 0.0.10 + image: api-server + imagePullPolicy: IfNotPresent + securePort: 8443 + storage: nacos + serverAddr: http://higress-nacos-service.higress-system.svc.cluster.local:8848 + username: "" + password: "" + namespaceID: "" + rsaKeyLength: 4096 # Skywalking config settings skywalking: @@ -646,3 +663,35 @@ skywalking: service: address: ~ port: 11800 + +# nacos config settings +nacos: + name: "higress-nacos" + enabled: false + hub: nacos/nacos-server + tag: latest + pullPolicy: IfNotPresent + replica: 1 + podManagementPolicy: Parallel + nodeSelector: { } + affinity: { } + tolerations: [ ] + persistence: + enabled: false + data: + accessModes: + - ReadWriteOnce + storageClassName: manual + resources: + requests: + storage: 5Gi + service: + port: 8848 + type: NodePort + nodePort: 30000 + annotations: {} + preferHostMode: hostname + resources: + requests: + cpu: 500m + memory: 2Gi diff --git a/test/e2e/conformance/base/nacos.yaml b/test/e2e/conformance/base/nacos.yaml index 459da47d40..c2e2c9e6d4 100644 --- a/test/e2e/conformance/base/nacos.yaml +++ b/test/e2e/conformance/base/nacos.yaml @@ -31,7 +31,7 @@ metadata: spec: containers: - name: nacos-standlone-rc3 - image: registry.cn-hangzhou.aliyuncs.com/hinsteny/nacos-standlone-rc3:1.0.0-RC3 + image: registry.cn-hangzhou.aliyuncs.com/hinsteny/nacos-standlone-rc3:1.0.0-RC3 ports: - containerPort: 8848 --- diff --git a/test/e2e/conformance/base/service.yaml b/test/e2e/conformance/base/service.yaml new file mode 100644 index 0000000000..67a964a277 --- /dev/null +++ b/test/e2e/conformance/base/service.yaml @@ -0,0 +1,104 @@ +# Copyright (c) 2022 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: infra-backend-v1 + namespace: higress-conformance-infra +spec: + selector: + app: infra-backend-v1 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: infra-backend-v2 + namespace: higress-conformance-infra +spec: + selector: + app: infra-backend-v2 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: infra-backend-v3 + namespace: higress-conformance-infra +spec: + selector: + app: infra-backend-v3 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: infra-backend-echo-body-v1 + namespace: higress-conformance-infra +spec: + selector: + app: infra-backend-echo-body-v1 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: app-backend-v1 + namespace: higress-conformance-app-backend +spec: + selector: + app: app-backend-v1 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: app-backend-v2 + namespace: higress-conformance-app-backend +spec: + selector: + app: app-backend-v2 + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 +--- +apiVersion: v1 +kind: Service +metadata: + name: web-backend + namespace: higress-conformance-web-backend +spec: + selector: + app: web-backend + ports: + - protocol: TCP + port: 8080 + targetPort: 3000 diff --git a/test/e2e/conformance/tests/configmap-global.go b/test/e2e/conformance/tests/configmap-global.go index a64043bbb4..5ae1bbe3ac 100644 --- a/test/e2e/conformance/tests/configmap-global.go +++ b/test/e2e/conformance/tests/configmap-global.go @@ -19,7 +19,6 @@ import ( "github.com/alibaba/higress/pkg/ingress/kube/configmap" "github.com/alibaba/higress/test/e2e/conformance/utils/envoy" - "github.com/alibaba/higress/test/e2e/conformance/utils/kubernetes" "github.com/alibaba/higress/test/e2e/conformance/utils/suite" ) @@ -1090,10 +1089,7 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ t.Run("ConfigMap Global Envoy", func(t *testing.T) { for _, testcase := range testCases { // apply config - err := kubernetes.ApplyConfigmapDataWithYaml(t, suite.Client, "higress-system", "higress-config", "higress", testcase.higressConfig) - if err != nil { - t.Fatalf("can't apply conifgmap %s in namespace %s for data key %s", "higress-config", "higress-system", "higress") - } + suite.Applier.MustApplyConfigmapDataWithYaml(t, suite.ConfigCenter, suite.Client, "higress-system", "higress-config", "higress", testcase.higressConfig, suite.EnableApiServer) t.Logf("Test Case %s", testcase.name) for _, assertion := range testcase.envoyAssertion { envoy.AssertEnvoyConfig(t, suite.TimeoutConfig, assertion) diff --git a/test/e2e/conformance/tests/configmap-gzip.go b/test/e2e/conformance/tests/configmap-gzip.go index 36caea2a60..2bf75ff2d6 100644 --- a/test/e2e/conformance/tests/configmap-gzip.go +++ b/test/e2e/conformance/tests/configmap-gzip.go @@ -20,7 +20,6 @@ import ( "github.com/alibaba/higress/pkg/ingress/kube/configmap" "github.com/alibaba/higress/test/e2e/conformance/utils/envoy" "github.com/alibaba/higress/test/e2e/conformance/utils/http" - "github.com/alibaba/higress/test/e2e/conformance/utils/kubernetes" "github.com/alibaba/higress/test/e2e/conformance/utils/suite" ) @@ -296,10 +295,7 @@ var ConfigmapGzip = suite.ConformanceTest{ Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { t.Run("Configmap Gzip", func(t *testing.T) { for _, testcase := range testCases { - err := kubernetes.ApplyConfigmapDataWithYaml(t, suite.Client, "higress-system", "higress-config", "higress", testcase.higressConfig) - if err != nil { - t.Fatalf("can't apply conifgmap %s in namespace %s for data key %s", "higress-config", "higress-system", "higress") - } + suite.Applier.MustApplyConfigmapDataWithYaml(t, suite.ConfigCenter, suite.Client, "higress-system", "higress-config", "higress", testcase.higressConfig, suite.EnableApiServer) http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, suite.GatewayAddress, testcase.httpAssert) } }) @@ -315,10 +311,7 @@ var ConfigMapGzipEnvoy = suite.ConformanceTest{ t.Run("ConfigMap Gzip Envoy", func(t *testing.T) { for _, testcase := range testCases { // apply config - err := kubernetes.ApplyConfigmapDataWithYaml(t, suite.Client, "higress-system", "higress-config", "higress", testcase.higressConfig) - if err != nil { - t.Fatalf("can't apply conifgmap %s in namespace %s for data key %s", "higress-config", "higress-system", "higress") - } + suite.Applier.MustApplyConfigmapDataWithYaml(t, suite.ConfigCenter, suite.Client, "higress-system", "higress-config", "higress", testcase.higressConfig, suite.EnableApiServer) envoy.AssertEnvoyConfig(t, suite.TimeoutConfig, testcase.envoyAssertion) } }) diff --git a/test/e2e/conformance/tests/cpp-wasm-basic-auth.go b/test/e2e/conformance/tests/cpp-wasm-basic-auth.go index 75d9753f6e..8ec8493f69 100644 --- a/test/e2e/conformance/tests/cpp-wasm-basic-auth.go +++ b/test/e2e/conformance/tests/cpp-wasm-basic-auth.go @@ -29,6 +29,7 @@ var CPPWasmPluginsBasicAuth = suite.ConformanceTest{ ShortName: "CPPWasmPluginsBasicAuth", Description: "The Ingress in the higress-conformance-infra namespace test the CPP basic-auth WASM plugin.", Manifests: []string{"tests/cpp-wasm-basic-auth.yaml"}, + PluginName: "basic-auth", Features: []suite.SupportedFeature{suite.WASMCPPConformanceFeature}, Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ diff --git a/test/e2e/conformance/tests/cpp-wasm-key-auth.go b/test/e2e/conformance/tests/cpp-wasm-key-auth.go index 1aeaa36dc0..d0ad8e83d0 100644 --- a/test/e2e/conformance/tests/cpp-wasm-key-auth.go +++ b/test/e2e/conformance/tests/cpp-wasm-key-auth.go @@ -29,6 +29,7 @@ var CPPWasmPluginsKeyAuth = suite.ConformanceTest{ ShortName: "CPPWasmPluginsKeyAuth", Description: "The Ingress in the higress-conformance-infra namespace test the CPP key_auth wasmplugins.", Manifests: []string{"tests/cpp-wasm-key-auth.yaml"}, + PluginName: "key-auth", Features: []suite.SupportedFeature{suite.WASMCPPConformanceFeature}, Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ diff --git a/test/e2e/conformance/tests/cpp-wasm-request-block.go b/test/e2e/conformance/tests/cpp-wasm-request-block.go index e00eba30f9..052cf18748 100644 --- a/test/e2e/conformance/tests/cpp-wasm-request-block.go +++ b/test/e2e/conformance/tests/cpp-wasm-request-block.go @@ -30,6 +30,7 @@ var CPPWasmPluginsRequestBlock = suite.ConformanceTest{ Description: "The Ingress in the higress-conformance-infra namespace test the cpp request-block wasmplugins.", Manifests: []string{"tests/cpp-wasm-request-block.yaml"}, Features: []suite.SupportedFeature{suite.WASMCPPConformanceFeature}, + PluginName: "request-block", Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ { diff --git a/test/e2e/conformance/tests/go-wasm-basic-auth.go b/test/e2e/conformance/tests/go-wasm-basic-auth.go index 3a67f61761..774615fc57 100644 --- a/test/e2e/conformance/tests/go-wasm-basic-auth.go +++ b/test/e2e/conformance/tests/go-wasm-basic-auth.go @@ -30,6 +30,7 @@ var WasmPluginsBasicAuth = suite.ConformanceTest{ Description: "The Ingress in the higress-conformance-infra namespace test the basic-auth WASM plugin.", Manifests: []string{"tests/go-wasm-basic-auth.yaml"}, Features: []suite.SupportedFeature{suite.WASMGoConformanceFeature}, + PluginName: "basic-auth", Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ { diff --git a/test/e2e/conformance/tests/go-wasm-key-auth.go b/test/e2e/conformance/tests/go-wasm-key-auth.go index 3adb7ecab6..d7e48462b9 100644 --- a/test/e2e/conformance/tests/go-wasm-key-auth.go +++ b/test/e2e/conformance/tests/go-wasm-key-auth.go @@ -30,6 +30,7 @@ var WasmPluginsKeyAuth = suite.ConformanceTest{ Description: "The Ingress in the higress-conformance-infra namespace test the key-auth WASM plugin.", Manifests: []string{"tests/go-wasm-key-auth.yaml"}, Features: []suite.SupportedFeature{suite.WASMGoConformanceFeature}, + PluginName: "key-auth", Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ { diff --git a/test/e2e/conformance/tests/go-wasm-request-block.go b/test/e2e/conformance/tests/go-wasm-request-block.go index 524205dbe0..a7e87255a0 100644 --- a/test/e2e/conformance/tests/go-wasm-request-block.go +++ b/test/e2e/conformance/tests/go-wasm-request-block.go @@ -30,6 +30,7 @@ var WasmPluginsRequestBlock = suite.ConformanceTest{ Description: "The Ingress in the higress-conformance-infra namespace test the request-block wasmplugins.", Manifests: []string{"tests/go-wasm-request-block.yaml"}, Features: []suite.SupportedFeature{suite.WASMGoConformanceFeature}, + PluginName: "request-block", Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ { diff --git a/test/e2e/conformance/tests/go-wasm-simple-jwt-auth.go b/test/e2e/conformance/tests/go-wasm-simple-jwt-auth.go index bf8d91434f..457ad234a4 100644 --- a/test/e2e/conformance/tests/go-wasm-simple-jwt-auth.go +++ b/test/e2e/conformance/tests/go-wasm-simple-jwt-auth.go @@ -30,6 +30,7 @@ var WasmPluginsJwtAuth = suite.ConformanceTest{ Description: "The Ingress in the higress-conformance-infra namespace test the simple-jwt-auth wasmplugins.", Manifests: []string{"tests/go-wasm-simple-jwt-auth.yaml"}, Features: []suite.SupportedFeature{suite.WASMGoConformanceFeature}, + PluginName: "simple-jwt-auth", Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ { diff --git a/test/e2e/conformance/tests/go-wasm-sni-misdirect.go b/test/e2e/conformance/tests/go-wasm-sni-misdirect.go index c921bd1e20..6880493dd0 100644 --- a/test/e2e/conformance/tests/go-wasm-sni-misdirect.go +++ b/test/e2e/conformance/tests/go-wasm-sni-misdirect.go @@ -33,6 +33,7 @@ var WasmPluginsSniMisdirect = suite.ConformanceTest{ Description: "The Ingress in the higress-conformance-infra namespace test the sni-misdirect wasmplugins.", Manifests: []string{"tests/go-wasm-sni-misdirect.yaml"}, Features: []suite.SupportedFeature{suite.WASMGoConformanceFeature}, + PluginName: "sni-misdirect", Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { // Prepare certificates and secrets for testcases caCertOut, _, caCert, caKey := cert.MustGenerateCaCert(t) diff --git a/test/e2e/conformance/tests/go-wasm-transformer.go b/test/e2e/conformance/tests/go-wasm-transformer.go index 30fe6321e2..14eea2e690 100644 --- a/test/e2e/conformance/tests/go-wasm-transformer.go +++ b/test/e2e/conformance/tests/go-wasm-transformer.go @@ -31,6 +31,7 @@ var WasmPluginsTransformer = suite.ConformanceTest{ Description: "The Ingress in the higress-conformance-infra namespace test the transformer WASM plugin.", Features: []suite.SupportedFeature{suite.WASMGoConformanceFeature}, Manifests: []string{"tests/go-wasm-transformer.yaml"}, + PluginName: "transformer", Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { testcases := []http.Assertion{ { diff --git a/test/e2e/conformance/utils/configcenter/nacos/nacos.go b/test/e2e/conformance/utils/configcenter/nacos/nacos.go new file mode 100644 index 0000000000..a7249fec92 --- /dev/null +++ b/test/e2e/conformance/utils/configcenter/nacos/nacos.go @@ -0,0 +1,101 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nacos + +import ( + "net/url" + "strconv" + "strings" + + cc "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter" + "github.com/nacos-group/nacos-sdk-go/v2/clients" + "github.com/nacos-group/nacos-sdk-go/v2/clients/config_client" + "github.com/nacos-group/nacos-sdk-go/v2/common/constant" + "github.com/nacos-group/nacos-sdk-go/v2/vo" +) + +type storage struct { + client config_client.IConfigClient +} + +func NewClient(addr string) (cc.Storage, error) { + clientConfig := constant.NewClientConfig( + constant.WithNamespaceId(""), + constant.WithUsername(""), + constant.WithPassword(""), + constant.WithLogLevel("info"), + ) + + serverUrl, err := url.Parse(addr) + if err != nil { + return nil, err + } + rawPort := serverUrl.Port() + var port uint64 + if rawPort != "" { + port, err = strconv.ParseUint(rawPort, 10, 0) + if err != nil || port < 1 || port > 65535 { + return nil, err + } + } else { + port = 80 + } + path := serverUrl.Path + if strings.HasSuffix(path, "/") { + path = path[:len(path)-1] + } + serverConfigs := []constant.ServerConfig{ + { + IpAddr: serverUrl.Hostname(), + ContextPath: path, + Port: port, + Scheme: serverUrl.Scheme, + }, + } + + client, err := clients.NewConfigClient( + vo.NacosClientParam{ + ClientConfig: clientConfig, + ServerConfigs: serverConfigs, + }, + ) + if err != nil { + return nil, err + } + return storage{ + client: client, + }, nil +} + +func (s storage) PublishConfig(kind, name, namespace, content string) error { + dataId := cc.GetDataId(kind, name) + group := namespace + _, err := s.client.PublishConfig(vo.ConfigParam{ + DataId: dataId, + Group: group, + Content: content, + }) + return err +} + +func (s storage) DeleteConfig(kind, name, namespace string) error { + dataId := cc.GetDataId(kind, name) + group := namespace + _, err := s.client.DeleteConfig(vo.ConfigParam{ + DataId: dataId, + Group: group, + }) + return err +} diff --git a/test/e2e/conformance/utils/configcenter/storage.go b/test/e2e/conformance/utils/configcenter/storage.go new file mode 100644 index 0000000000..dc64350eae --- /dev/null +++ b/test/e2e/conformance/utils/configcenter/storage.go @@ -0,0 +1,45 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cc + +import "strings" + +type Storage interface { + PublishConfig(kind, name, namespace, content string) error + DeleteConfig(kind, name, namespace string) error +} + +func GetDataId(kind, name string) string { + switch strings.ToLower(kind) { + case "configmap": + kind = "configmaps" + case "secret": + kind = "secrets" + case "ingress": + kind = "ingresses" + case "service": + kind = "services" + case "ingressclass": + kind = "ingressclasses" + case "mcpbridge": + kind = "mcpbridges" + case "wasmplugin": + kind = "wasmplugins" + case "http2rpc": + kind = "http2rpcs" + default: + } + return kind + "." + name +} diff --git a/test/e2e/conformance/utils/flags/flags.go b/test/e2e/conformance/utils/flags/flags.go index 9b2d4317ce..d13db59cb2 100644 --- a/test/e2e/conformance/utils/flags/flags.go +++ b/test/e2e/conformance/utils/flags/flags.go @@ -25,7 +25,10 @@ var ( ExemptFeatures = flag.String("exempt-features", "", "Exempt Features excluded from conformance tests suites") IsWasmPluginTest = flag.Bool("isWasmPluginTest", false, "Determine if run wasm plugin conformance test") WasmPluginType = flag.String("wasmPluginType", "GO", "Define wasm plugin type, currently supports GO, CPP") - WasmPluginName = flag.String("wasmPluginName", "", "Define wasm plugin name") + WasmPluginName = flag.String("wasmPluginName", "all", "Define wasm plugin name") IsEnvoyConfigTest = flag.Bool("isEnvoyConfigTest", false, "Determine if run envoy config conformance test") TestArea = flag.String("test-area", "all", "Test area to run, like all to run setup/run/clean, setup to prepare test environment, run to run test cases, clean to clean test environment") + EnableApiServer = flag.Bool("enableApiServer", false, "Determine if enable api server") + // TODO when apiserver support other mode, we can use string field like: nacos://username:password@ip:port/namespace + ApiServerStorage = flag.String("storage", "http://127.0.0.1:8848", "Define storage") ) diff --git a/test/e2e/conformance/utils/kubernetes/apply.go b/test/e2e/conformance/utils/kubernetes/apply.go index b552e85065..98b28bde26 100644 --- a/test/e2e/conformance/utils/kubernetes/apply.go +++ b/test/e2e/conformance/utils/kubernetes/apply.go @@ -22,14 +22,18 @@ import ( "net/http" "strings" "testing" + "time" ingress "github.com/alibaba/higress/test/e2e/conformance" + cc "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter" "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/controller-runtime/pkg/client" + yamlFromK8s "sigs.k8s.io/yaml" "github.com/alibaba/higress/test/e2e/conformance/utils/config" ) @@ -249,3 +253,90 @@ func getContentsFromPathOrURL(location string, timeoutConfig config.TimeoutConfi } return bytes.NewBuffer(b), nil } + +// MustPublishConfig publish config to config center +func (a Applier) MustPublishConfig(t *testing.T, timeoutConfig config.TimeoutConfig, location string, cleanup bool, cc cc.Storage) { + data, err := getContentsFromPathOrURL(location, timeoutConfig) + require.NoError(t, err) + + decoder := yaml.NewYAMLOrJSONDecoder(data, 4096) + + resources, err := a.prepareResources(t, decoder) + if err != nil { + t.Logf("🧳 Manifest: %s", data.String()) + require.NoErrorf(t, err, "error parsing manifest") + } + + for i := range resources { + r := resources[i] + var content []byte + content, err = r.MarshalJSON() + require.NoError(t, err) + // publish + err = cc.PublishConfig(r.GetKind(), r.GetName(), r.GetNamespace(), string(content)) + require.NoError(t, err) + if cleanup { + t.Cleanup(func() { + // delete + t.Logf("🚮 Deleting %s %s", r.GetName(), r.GetKind()) + err = cc.DeleteConfig(r.GetKind(), r.GetName(), r.GetNamespace()) + require.NoError(t, err) + }) + } + } +} + +// MustDeleteConfig delete config from config center +func (a Applier) MustDeleteConfig(t *testing.T, timeoutConfig config.TimeoutConfig, location string, cc cc.Storage) { + data, err := getContentsFromPathOrURL(location, timeoutConfig) + require.NoError(t, err) + + decoder := yaml.NewYAMLOrJSONDecoder(data, 4096) + + resources, err := a.prepareResources(t, decoder) + if err != nil { + t.Logf("🧳 Manifest: %s", data.String()) + require.NoErrorf(t, err, "error parsing manifest") + } + + for i := range resources { + r := resources[i] + t.Logf("🚮 Deleting %s %s", r.GetName(), r.GetKind()) + err = cc.DeleteConfig(r.GetKind(), r.GetName(), r.GetNamespace()) + require.NoError(t, err) + } +} + +// MustApplyConfigmapDataWithYaml apply configmap data with yaml +func (a Applier) MustApplyConfigmapDataWithYaml(t *testing.T, cc cc.Storage, c client.Client, namespace string, name string, key string, val any, enableApiServer bool) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + cm := &v1.ConfigMap{} + err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, cm) + require.NoError(t, err) + + y, err := yamlFromK8s.Marshal(val) + require.NoError(t, err) + data := string(y) + + if cm.Data == nil { + cm.Data = make(map[string]string, 0) + } + cm.Data[key] = data + + t.Logf("🏗 Updating %s %s", name, namespace) + + if enableApiServer { + marshal, err := yamlFromK8s.Marshal(cm) + require.NoError(t, err) + err = cc.PublishConfig("configmap", cm.GetName(), cm.GetNamespace(), string(marshal)) + require.NoError(t, err) + return + } + + if err := c.Update(ctx, cm); err != nil { + require.NoError(t, err) + } + +} diff --git a/test/e2e/conformance/utils/kubernetes/helpers.go b/test/e2e/conformance/utils/kubernetes/helpers.go index f423e1d680..fd91c9a617 100644 --- a/test/e2e/conformance/utils/kubernetes/helpers.go +++ b/test/e2e/conformance/utils/kubernetes/helpers.go @@ -19,8 +19,6 @@ import ( "testing" "time" - "sigs.k8s.io/yaml" - "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -121,26 +119,3 @@ func FindPodConditionInList(t *testing.T, conditions []v1.PodCondition, condName t.Logf("⌛️ %s was not in conditions list", condName) return false } - -func ApplyConfigmapDataWithYaml(t *testing.T, c client.Client, namespace string, name string, key string, val any) error { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - cm := &v1.ConfigMap{} - if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, cm); err != nil { - return err - } - y, err := yaml.Marshal(val) - if err != nil { - return err - } - data := string(y) - - if cm.Data == nil { - cm.Data = make(map[string]string, 0) - } - cm.Data[key] = data - - t.Logf("🏗 Updating %s %s", name, namespace) - return c.Update(ctx, cm) -} diff --git a/test/e2e/conformance/utils/suite/suite.go b/test/e2e/conformance/utils/suite/suite.go index f958084a4e..fc1c428042 100644 --- a/test/e2e/conformance/utils/suite/suite.go +++ b/test/e2e/conformance/utils/suite/suite.go @@ -18,9 +18,12 @@ import ( "testing" "github.com/alibaba/higress/test/e2e/conformance/utils/config" + cc "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter" + "github.com/alibaba/higress/test/e2e/conformance/utils/configcenter/nacos" "github.com/alibaba/higress/test/e2e/conformance/utils/kubernetes" "github.com/alibaba/higress/test/e2e/conformance/utils/roundtripper" "istio.io/istio/pilot/pkg/util/sets" + "istio.io/pkg/log" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -45,6 +48,10 @@ type ConformanceTestSuite struct { SkipTests sets.Set TimeoutConfig config.TimeoutConfig SupportedFeatures sets.Set + EnableApiServer bool + ApiServerStorage string + ConfigCenter cc.Storage + WASMOptions } // Options can be used to initialize a ConformanceTestSuite. @@ -70,6 +77,9 @@ type Options struct { // IsEnvoyConfigTest indicates whether or not the test is for envoy config IsEnvoyConfigTest bool + + EnableApiServer bool + ApiServerStorage string } type WASMOptions struct { @@ -119,7 +129,10 @@ func New(s Options) *ConformanceTestSuite { Applier: kubernetes.Applier{ NamespaceLabels: s.NamespaceLabels, }, - TimeoutConfig: s.TimeoutConfig, + TimeoutConfig: s.TimeoutConfig, + EnableApiServer: s.EnableApiServer, + ApiServerStorage: s.ApiServerStorage, + WASMOptions: s.WASMOptions, } // apply defaults @@ -134,6 +147,14 @@ func New(s Options) *ConformanceTestSuite { } } + if suite.EnableApiServer { + configClient, err := nacos.NewClient(suite.ApiServerStorage) + if err != nil { + log.Fatalf("🚨 Failed to create config client: %v", err) + } + suite.ConfigCenter = configClient + } + return suite } @@ -150,6 +171,11 @@ func (suite *ConformanceTestSuite) Setup(t *testing.T) { suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, baseManifest, suite.Cleanup) } + if suite.EnableApiServer { + t.Logf("📦 Test Setup: Applying ApiServer Storage Manifests") + suite.Applier.MustPublishConfig(t, suite.TimeoutConfig, "base/service.yaml", suite.Cleanup, suite.ConfigCenter) + } + t.Logf("📦 Test Setup: Applying programmatic resources") secret := kubernetes.MustCreateSelfSignedCertSecret(t, "higress-conformance-web-backend", "certificate", []string{"*"}) suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup) @@ -205,6 +231,7 @@ type ConformanceTest struct { Description string PreDeleteRs []string Manifests []string + PluginName string Features []SupportedFeature Slow bool Parallel bool @@ -232,16 +259,35 @@ func (test *ConformanceTest) Run(t *testing.T, suite *ConformanceTestSuite) { t.Skipf("🏊🏼 Skipping %s: test explicitly skipped", test.ShortName) } + // Skip wasm plugin name test + if suite.IsWasmPluginTest { + if suite.WasmPluginName != "all" && suite.WasmPluginName != test.PluginName { + t.Skipf("🏊🏼 Skipping %s: wasm plugin name not match", test.ShortName) + } + } + t.Logf("🔥 Running Conformance Test: %s", test.ShortName) - for _, manifestLocation := range test.PreDeleteRs { - t.Logf("🧳 Applying PreDeleteRs Manifests: %s", manifestLocation) - suite.Applier.MustDelete(t, suite.Client, suite.TimeoutConfig, manifestLocation) - } + if suite.EnableApiServer { + for _, manifestLocation := range test.PreDeleteRs { + t.Logf("🧳 Applying PreDeleteRs ApiServer Storage Manifests: %s", manifestLocation) + suite.Applier.MustDeleteConfig(t, suite.TimeoutConfig, manifestLocation, suite.ConfigCenter) + } - for _, manifestLocation := range test.Manifests { - t.Logf("🧳 Applying Manifests: %s", manifestLocation) - suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, !test.NotCleanup) + for _, manifestLocation := range test.Manifests { + t.Logf("🧳 Applying ApiServer Storage Manifests: %s", manifestLocation) + suite.Applier.MustPublishConfig(t, suite.TimeoutConfig, manifestLocation, !test.NotCleanup, suite.ConfigCenter) + } + } else { + for _, manifestLocation := range test.PreDeleteRs { + t.Logf("🧳 Applying PreDeleteRs Manifests: %s", manifestLocation) + suite.Applier.MustDelete(t, suite.Client, suite.TimeoutConfig, manifestLocation) + } + + for _, manifestLocation := range test.Manifests { + t.Logf("🧳 Applying Manifests: %s", manifestLocation) + suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, !test.NotCleanup) + } } test.Test(t, suite) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 007fe7818c..16e05c4dbd 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -52,6 +52,8 @@ func TestHigressConformanceTests(t *testing.T) { }, GatewayAddress: "localhost", EnableAllSupportedFeatures: true, + EnableApiServer: *flags.EnableApiServer, + ApiServerStorage: *flags.ApiServerStorage, IsEnvoyConfigTest: *flags.IsEnvoyConfigTest, }) diff --git a/tools/hack/gen-secret-configmap.sh b/tools/hack/gen-secret-configmap.sh new file mode 100755 index 0000000000..bf82c76db9 --- /dev/null +++ b/tools/hack/gen-secret-configmap.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +# Copyright (c) 2022 Alibaba Group Holding Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http:www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +VOLUMES_ROOT="/tmp/higress-apiserver" +RSA_KEY_LENGTH=4096 +NAMESPACE="${NAMESPACE:-higress-system}" +CLUSTER_NAME="${CLUSTER_NAME:-higress}" +APISERVER_ADDRESS="${APISERVER_ADDRESS:-https://127.0.0.1:8443}" + +initializeApiServer() { + echo "Initializing API server configurations..." + + mkdir -p "$VOLUMES_ROOT/api" && cd "$_" + checkExitCode "Creating volume for API server fails with $?" + + if [ ! -f ca.key ] || [ ! -f ca.crt ]; then + echo " Generating CA certificate..."; + openssl req -nodes -new -x509 -days 36500 -keyout ca.key -out ca.crt -subj "/CN=higress-root-ca/O=higress" > /dev/null 2>&1 + checkExitCode " Generating CA certificate for API server fails with $?"; + else + echo " CA certificate already exists."; + fi + + if [ ! -f server.key ] || [ ! -f server.crt ]; then + echo " Generating server certificate..." + openssl req -out server.csr -new -newkey rsa:$RSA_KEY_LENGTH -nodes -keyout server.key -subj "/CN=higress-api-server/O=higress" > /dev/null 2>&1 \ + && openssl x509 -req -days 36500 -in server.csr -CA ca.crt -CAkey ca.key -set_serial 01 -sha256 -out server.crt > /dev/null 2>&1 + checkExitCode " Generating server certificate fails with $?"; + else + echo " Server certificate already exists."; + fi + + if [ ! -f nacos.key ]; then + echo " Generating data encryption key..." + if [ -z "$NACOS_DATA_ENC_KEY" ]; then + cat /dev/urandom | tr -dc '[:graph:]' | head -c 32 > nacos.key + else + echo -n "$NACOS_DATA_ENC_KEY" > nacos.key + fi + else + echo " Client certificate already exists."; + fi + + if [ ! -f client.key ] || [ ! -f client.crt ]; then + echo " Generating client certificate..." + openssl req -out client.csr -new -newkey rsa:$RSA_KEY_LENGTH -nodes -keyout client.key -subj "/CN=higress/O=system:masters" > /dev/null 2>&1 \ + && openssl x509 -req -days 36500 -in client.csr -CA ca.crt -CAkey ca.key -set_serial 02 -sha256 -out client.crt > /dev/null 2>&1 + checkExitCode " Generating client certificate fails with $?"; + else + echo " Client certificate already exists."; + fi +} + +applySecretConfigmap() { + # create namespace if not exists + kubectl get namespace $NAMESPACE > /dev/null 2>&1 + if [ $? -ne 0 ]; then + echo "Creating namespace $NAMESPACE..." + kubectl create namespace $NAMESPACE + checkExitCode "Creating namespace fails with $?" + fi + + echo "Applying secret $NAMESPACE/higress-apiserver..." + kubectl apply -f - <