From 151061773fce9ffe9e40184ab32887d1dbe00302 Mon Sep 17 00:00:00 2001 From: Hinsteny Date: Mon, 19 Jun 2023 14:37:37 +0800 Subject: [PATCH] feat: Support run e2e test casese in local Signed-off-by: Hinsteny --- Makefile | 34 ++++++++++++++++++- tools/hack/create-cluster.sh | 26 +++++++++++++++ tools/hack/install-kruise.sh | 49 ++++++++++++++++++++++++++++ tools/hack/kind-load-image.sh | 54 +++++++++++++++++++++++++++++++ tools/hack/run-kruise-e2e-test.sh | 32 ++++++++++++++++++ tools/src/kind/go.mod | 22 +++++++++++++ tools/src/kind/go.sum | 46 ++++++++++++++++++++++++++ tools/src/kind/pin.go | 20 ++++++++++++ tools/tools.mk | 17 ++++++++++ 9 files changed, 299 insertions(+), 1 deletion(-) create mode 100755 tools/hack/create-cluster.sh create mode 100755 tools/hack/install-kruise.sh create mode 100755 tools/hack/kind-load-image.sh create mode 100755 tools/hack/run-kruise-e2e-test.sh create mode 100644 tools/src/kind/go.mod create mode 100644 tools/src/kind/go.sum create mode 100644 tools/src/kind/pin.go create mode 100644 tools/tools.mk diff --git a/Makefile b/Makefile index ebb12b1ade..a14728476a 100644 --- a/Makefile +++ b/Makefile @@ -109,8 +109,40 @@ set -e ;\ TMP_DIR=$$(mktemp -d) ;\ cd $$TMP_DIR ;\ go mod init tmp ;\ -echo "Downloading $(2)" ;\ +echo "Downloading $(2) to $(PROJECT_DIR)/bin" ;\ GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\ rm -rf $$TMP_DIR ;\ } endef + +# create-cluster creates a kube cluster with kind. +.PHONY: create-cluster +create-cluster: $(tools/kind) + tools/hack/create-cluster.sh + +# delete-cluster deletes a kube cluster. +.PHONY: delete-cluster +delete-cluster: $(tools/kind) ## Delete kind cluster. + $(tools/kind) delete cluster --name ci-testing + +# kube-load-image loads a local built docker image into kube cluster. +.PHONY: kube-load-image +kube-load-image: $(tools/kind) + tools/hack/kind-load-image.sh $(IMG) + +# install-kruise install kruise with local build image to kube cluster. +.PHONY: install-kruise +install-kruise: + tools/hack/install-kruise.sh $(IMG) + +# run-kruise-e2e-test starts to run kruise e2e tests. +.PHONY: run-kruise-e2e-test +run-kruise-e2e-test: + @echo -e "\n\033[36mRunning kruise e2e tests...\033[0m" + tools/hack/run-kruise-e2e-test.sh + +include tools/tools.mk + +# kruise-e2e-test runs kruise e2e tests. +.PHONY: kruise-e2e-test +kruise-e2e-test: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-kruise run-kruise-e2e-test delete-cluster diff --git a/tools/hack/create-cluster.sh b/tools/hack/create-cluster.sh new file mode 100755 index 0000000000..5f16afe5b6 --- /dev/null +++ b/tools/hack/create-cluster.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright (c) 2023 Alibaba Group Holding Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +# Setup default values +CLUSTER_NAME=${CLUSTER_NAME:-"ci-testing"} +KIND_NODE_TAG=${KIND_NODE_TAG:-"v1.24.2"} + +echo "$KIND_NODE_TAG" +echo "$CLUSTER_NAME" + +## Create kind cluster. +tools/bin/kind create cluster --image "kindest/node:${KIND_NODE_TAG}" --name "${CLUSTER_NAME}" --config=test/kind-conf.yaml diff --git a/tools/hack/install-kruise.sh b/tools/hack/install-kruise.sh new file mode 100755 index 0000000000..64133bbb96 --- /dev/null +++ b/tools/hack/install-kruise.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Copyright (c) 2023 Alibaba Group Holding Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +readonly IMAGE="$1" + +export GOPROXY=https://mirrors.aliyun.com/goproxy/ + +kubectl cluster-info + +IMG=${IMAGE} ./scripts/deploy_kind.sh + +NODES=$(kubectl get node | wc -l) +for ((i=1;i<10;i++)); +do + set +e + PODS=$(kubectl get pod -n kruise-system | grep -c '1/1') + set -e + if [ "$PODS" -eq "$NODES" ]; then + break + fi + sleep 3 +done +set +e +PODS=$(kubectl get pod -n kruise-system | grep -c '1/1') +kubectl get node -o yaml +kubectl get all -n kruise-system -o yaml +kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system +kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true +set -e +if [ "$PODS" -eq "$NODES" ]; then + echo "Wait for kruise-manager and kruise-daemon ready successfully" +else + echo "Timeout to wait for kruise-manager and kruise-daemon ready" + exit 1 +fi \ No newline at end of file diff --git a/tools/hack/kind-load-image.sh b/tools/hack/kind-load-image.sh new file mode 100755 index 0000000000..0a5e66b309 --- /dev/null +++ b/tools/hack/kind-load-image.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Copyright (c) 2023 Alibaba Group Holding Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +readonly KIND=${KIND:-tools/bin/kind} +readonly CLUSTER_NAME=${CLUSTER_NAME:-"ci-testing"} + +# Docker variables +readonly IMAGE="$1" + +# Wrap sed to deal with GNU and BSD sed flags. +run::sed() { + if sed --version &1 | grep -q GNU; then + # GNU sed + sed -i "$@" + else + # assume BSD sed + sed -i '' "$@" + fi +} + +kind::cluster::exists() { + ${KIND} get clusters | grep -q "$1" +} + +kind::cluster::load() { + ${KIND} load docker-image \ + --name "${CLUSTER_NAME}" \ + "$@" +} + +if ! kind::cluster::exists "$CLUSTER_NAME" ; then + echo "cluster $CLUSTER_NAME does not exist" + exit 2 +fi + +# Push the Higress image to the kind cluster. +echo "Loading image ${IMAGE} to kind cluster ${CLUSTER_NAME}..." +kind::cluster::load "${IMAGE}" diff --git a/tools/hack/run-kruise-e2e-test.sh b/tools/hack/run-kruise-e2e-test.sh new file mode 100755 index 0000000000..d65fd61060 --- /dev/null +++ b/tools/hack/run-kruise-e2e-test.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copyright (c) 2023 Alibaba Group Holding Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +export KUBECONFIG=${HOME}/.kube/config +make ginkgo +set +e +./bin/ginkgo -timeout 60m -v --focus='\[apps\] StatefulSet' test/e2e +retVal=$? +restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}') +if [ "${restartCount}" -eq "0" ];then + echo "Kruise-manager has not restarted" +else + kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers + echo "Kruise-manager has restarted, abort!!!" + kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system + exit 1 +fi +exit $retVal \ No newline at end of file diff --git a/tools/src/kind/go.mod b/tools/src/kind/go.mod new file mode 100644 index 0000000000..c2689a5601 --- /dev/null +++ b/tools/src/kind/go.mod @@ -0,0 +1,22 @@ +module github.com/envoyproxy/gateway/tools/src/kind + +go 1.19 + +require sigs.k8s.io/kind v0.17.0 + +require ( + github.com/BurntSushi/toml v1.0.0 // indirect + github.com/alessio/shellescape v1.4.1 // indirect + github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/spf13/cobra v1.4.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) diff --git a/tools/src/kind/go.sum b/tools/src/kind/go.sum new file mode 100644 index 0000000000..25b1da2e36 --- /dev/null +++ b/tools/src/kind/go.sum @@ -0,0 +1,46 @@ +github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= +github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= +github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 h1:SJ+NtwL6QaZ21U+IrK7d0gGgpjGGvd2kz+FzTHVzdqI= +github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2/go.mod h1:Tv1PlzqC9t8wNnpPdctvtSUOPUUg4SHeE6vR1Ir2hmg= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/kind v0.14.0 h1:cNmI3jGBvp7UegEGbC5we8plDtCUmaNRL+bod7JoSCE= +sigs.k8s.io/kind v0.14.0/go.mod h1:UrFRPHG+2a5j0Q7qiR4gtJ4rEyn8TuMQwuOPf+m4oHg= +sigs.k8s.io/kind v0.17.0 h1:CScmGz/wX66puA06Gj8OZb76Wmk7JIjgWf5JDvY7msM= +sigs.k8s.io/kind v0.17.0/go.mod h1:Qqp8AiwOlMZmJWs37Hgs31xcbiYXjtXlRBSftcnZXQk= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/tools/src/kind/pin.go b/tools/src/kind/pin.go new file mode 100644 index 0000000000..afafdb5410 --- /dev/null +++ b/tools/src/kind/pin.go @@ -0,0 +1,20 @@ +// Copyright (c) 2023 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build pin +// +build pin + +package ignore + +import "sigs.k8s.io/kind" diff --git a/tools/tools.mk b/tools/tools.mk new file mode 100644 index 0000000000..d54ef78a30 --- /dev/null +++ b/tools/tools.mk @@ -0,0 +1,17 @@ +tools.bindir = tools/bin +tools.srcdir = tools/src + +ifeq ($(origin GOOS), undefined) + GOOS := $(shell go env GOOS) +endif +ifeq ($(origin GOARCH), undefined) + GOARCH := $(shell go env GOARCH) +endif + + +# `go get`-able things +# ==================== +# +tools/kind = $(tools.bindir)/kind +$(tools.bindir)/%: $(tools.srcdir)/%/pin.go $(tools.srcdir)/%/go.mod + cd $(