Merge pull request #689 from village-way/main

feature: enhance vip and resolve conflict when merge release-0.4.0 to main

cmd/kubenest/node-agent/app/client/client.go

@@ -60,7 +60,7 @@ var (
 	}
 	wg sync.WaitGroup
 
-	wsAddr    []string // websocket client connect address list
+	WsAddr    []string // websocket client connect address list
 	filePath  string   // the server path to save upload file
 	fileName  string   // local file to upload
 	params    []string // New slice to hold multiple command parameters
@@ -79,7 +79,7 @@ func cmdCheckRun(cmd *cobra.Command, args []string) error {
 	headers := http.Header{
 		"Authorization": {"Basic " + auth},
 	}
-	for _, addr := range wsAddr {
+	for _, addr := range WsAddr {
 		wg.Add(1)
 		go func(addr string) {
 			defer wg.Done()
@@ -106,15 +106,11 @@ func init() {
 	// #nosec G402
 	dialer.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
 
-	ClientCmd.PersistentFlags().StringSliceVarP(&wsAddr, "addr", "a", []string{}, "WebSocket address (e.g., host1:port1,host2:port2)")
-	err := ClientCmd.MarkPersistentFlagRequired("addr")
-	if err != nil {
-		return
-	}
+	ClientCmd.PersistentFlags().StringSliceVarP(&WsAddr, "addr", "a", []string{}, "WebSocket address (e.g., host1:port1,host2:port2)")
 
 	// PreRunE check param
 	ClientCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {
-		for _, value := range wsAddr {
+		for _, value := range WsAddr {
 			if _, exists := uniqueValuesMap[value]; exists {
 				return errors.New("duplicate values are not allowed")
 			}
@@ -137,7 +133,7 @@ func init() {
 	_ = uploadCmd.MarkFlagRequired("path")
 
 	ttyCmd.Flags().StringVarP(&operation, "operation", "o", "", "Operation to perform")
-	err = ttyCmd.MarkFlagRequired("operation") // Ensure 'operation' flag is required for ttyCmd
+	err := ttyCmd.MarkFlagRequired("operation") // Ensure 'operation' flag is required for ttyCmd
 	if err != nil {
 		return
 	}
@@ -157,7 +153,7 @@ func cmdTtyRun(cmd *cobra.Command, args []string) error {
 	}
 	cmdStr := fmt.Sprintf("command=%s", operation)
 	// execute one every wsAddr
-	for _, addr := range wsAddr {
+	for _, addr := range WsAddr {
 		wsURL := fmt.Sprintf("wss://%s/tty/?%s", addr, cmdStr)
 		fmt.Println("Executing tty:", cmdStr, "on", addr)
 		err := connectTty(wsURL, headers)
@@ -294,7 +290,7 @@ func executeWebSocketCommand(auth string) error {
 	}
 
 	// execute one every wsAddr
-	for _, addr := range wsAddr {
+	for _, addr := range WsAddr {
 		wg.Add(1)
 		go func(addr string) {
 			defer wg.Done()
@@ -314,7 +310,7 @@ func uploadFile(filePath, fileName, auth string) error {
 	headers := http.Header{
 		"Authorization": {"Basic " + auth},
 	}
-	for _, addr := range wsAddr {
+	for _, addr := range WsAddr {
 		wg.Add(1)
 		go func(addr string) {
 			defer wg.Done()

cmd/kubenest/node-agent/app/root.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"os"
+	"strings"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -33,6 +34,7 @@ func initConfig() {
 	currentDir, _ := os.Getwd()
 	viper.AddConfigPath(currentDir)
 	viper.AddConfigPath("/srv/node-agent/agent.env")
+	viper.SetConfigType("toml")
 	// If a agent.env file is found, read it in.
 	if err := viper.ReadInConfig(); err != nil {
 		log.Warnf("Load config file error, %s", err)
@@ -46,8 +48,27 @@ func initConfig() {
 	}
 }
 
+func initWebSocketAddr() {
+	err := viper.BindPFlag("ADDR", client.ClientCmd.PersistentFlags().Lookup("addr"))
+	if err != nil {
+		log.Fatalf("Failed to bind flag: %v", err)
+		return
+	}
+	err = viper.BindEnv("ADDR", "ADDR")
+	if err != nil {
+		log.Fatalf("Failed to bind env: %v", err)
+		return
+	}
+	// Initialize addr value from viper
+	log.Infof(strings.Join(viper.AllKeys(), ","))
+	if viper.Get("addr") != nil {
+		client.WsAddr = viper.GetStringSlice("addr")
+		log.Infof("addr: %v", client.WsAddr)
+	}
+}
+
 func init() {
-	cobra.OnInitialize(initConfig)
+	cobra.OnInitialize(initConfig, initWebSocketAddr)
 
 	RootCmd.PersistentFlags().StringVarP(&user, "user", "u", "", "Username for authentication")
 	RootCmd.PersistentFlags().StringVarP(&password, "password", "p", "", "Password for authentication")

deploy/crds/kosmos.io_kubenestconfigurations.yaml

@@ -52,7 +52,25 @@ spec:
               etcdUnitSize:
                 type: string
               forceDestroy:
+                description: todo Group according to the parameters of apiserver,
+                  etcd, coredns, etc. ForceDestroy indicates whether to force destroy
+                  the cluster.
                 type: boolean
+              tenantEntrypoint:
+                description: TenantEntrypoint TenantEntrypoint `yaml:"tenantEntrypoint"
+                  json:"tenantEntrypoint,omitempty"`
+                properties:
+                  externalIps:
+                    description: ExternalIP is the external ip of the tenant entrypoint.
+                    items:
+                      type: string
+                    type: array
+                  externalVips:
+                    description: ExternalVips is the external vips of the tenant entrypoint.
+                    items:
+                      type: string
+                    type: array
+                type: object
             type: object
           kubeNestType:
             type: string

deploy/crds/kosmos.io_virtualclusters.yaml

@@ -53,6 +53,43 @@ spec:
                 items:
                   type: string
                 type: array
+              kubeInKubeConfig:
+                description: KubeInKubeConfig is the external config of virtual cluster
+                properties:
+                  admissionPlugins:
+                    type: boolean
+                  anpMode:
+                    type: string
+                  apiServerReplicas:
+                    type: integer
+                  clusterCIDR:
+                    type: string
+                  etcdStorageClass:
+                    type: string
+                  etcdUnitSize:
+                    type: string
+                  forceDestroy:
+                    description: todo Group according to the parameters of apiserver,
+                      etcd, coredns, etc. ForceDestroy indicates whether to force
+                      destroy the cluster.
+                    type: boolean
+                  tenantEntrypoint:
+                    description: TenantEntrypoint TenantEntrypoint `yaml:"tenantEntrypoint"
+                      json:"tenantEntrypoint,omitempty"`
+                    properties:
+                      externalIps:
+                        description: ExternalIP is the external ip of the tenant entrypoint.
+                        items:
+                          type: string
+                        type: array
+                      externalVips:
+                        description: ExternalVips is the external vips of the tenant
+                          entrypoint.
+                        items:
+                          type: string
+                        type: array
+                    type: object
+                type: object
               kubeconfig:
                 description: Kubeconfig is the kubeconfig of the virtual kubernetes's
                   control plane
@@ -208,6 +245,10 @@ spec:
               updateTime:
                 format: date-time
                 type: string
+              vipMap:
+                additionalProperties:
+                  type: string
+                type: object
             type: object
         required:
         - spec

deploy/virtual-cluster-components-manifest-cm.yaml

@@ -4,6 +4,7 @@ data:
     [
       {"name": "kube-proxy", "path": "/kosmos/manifest/kube-proxy/*.yaml"},
       {"name": "calico", "path": "/kosmos/manifest/calico/*.yaml"},
+      {"name": "keepalived", "path": "/kosmos/manifest/keepalived/*.yaml"},
     ]
   host-core-dns-components: |
     [

deploy/virtual-cluster-vip-pool-cm.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kosmos-vip-pool
+  namespace: kosmos-system
+data:
+  vip-config.yaml: |
+    # can be use for vc, the ip formate is 192.168.0.1 and 192.168.0.2-192.168.0.10
+    vipPool:
+      - 192.168.0.1-192.168.0.10

hack/node-agent/init.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
-WEB_USER="$WEB_USER" sed -i 's/^WEB_USER=.*/WEB_USER='"$WEB_USER"'/' /app/agent.env
-WEB_PASS="$WEB_PASS" sed -i 's/^WEB_PASS=.*/WEB_PASS='"$WEB_PASS"'/' /app/agent.env
+WEB_USER="$WEB_USER" sed -i 's/^WEB_USER=.*/WEB_USER="'"$WEB_USER"'"/' /app/agent.env
+WEB_PASS="$WEB_PASS" sed -i 's/^WEB_PASS=.*/WEB_PASS="'"$WEB_PASS"'"/' /app/agent.env
 sha256sum /app/node-agent > /app/node-agent.sum
 sha256sum /host-path/node-agent >> /app/node-agent.sum
 rsync -avz /app/ /host-path/

pkg/apis/kosmos/v1alpha1/kubenestconfiguration_types.go

@@ -36,14 +36,21 @@ type KosmosKubeConfig struct {
 
 type KubeInKubeConfig struct {
 	// todo Group according to the parameters of apiserver, etcd, coredns, etc.
-
-	ForceDestroy      bool   `yaml:"forceDestroy" json:"forceDestroy,omitempty"`
-	AnpMode           string `yaml:"anpMode" json:"anpMode,omitempty"`
-	AdmissionPlugins  bool   `yaml:"admissionPlugins" json:"admissionPlugins,omitempty"`
-	ApiServerReplicas int    `yaml:"apiServerReplicas" json:"apiServerReplicas,omitempty"`
-	ClusterCIDR       string `yaml:"clusterCIDR" json:"clusterCIDR,omitempty"`
-	ETCDStorageClass  string `yaml:"etcdStorageClass" json:"etcdStorageClass,omitempty"`
-	ETCDUnitSize      string `yaml:"etcdUnitSize" json:"etcdUnitSize,omitempty"`
+	// ForceDestroy indicates whether to force destroy the cluster.
+	// +optional
+	ForceDestroy bool `yaml:"forceDestroy" json:"forceDestroy,omitempty"`
+	// +optional
+	AnpMode string `yaml:"anpMode" json:"anpMode,omitempty"`
+	// +optional
+	AdmissionPlugins bool `yaml:"admissionPlugins" json:"admissionPlugins,omitempty"`
+	// +optional
+	ApiServerReplicas int `yaml:"apiServerReplicas" json:"apiServerReplicas,omitempty"`
+	// +optional
+	ClusterCIDR string `yaml:"clusterCIDR" json:"clusterCIDR,omitempty"`
+	// +optional
+	ETCDStorageClass string `yaml:"etcdStorageClass" json:"etcdStorageClass,omitempty"`
+	// +optional
+	ETCDUnitSize string `yaml:"etcdUnitSize" json:"etcdUnitSize,omitempty"`
 
 	//// Etcd contains the configuration for the etcd statefulset.
 	//Etcd EtcdCluster `yaml:"etcd" json:"etcd,omitempty"`
@@ -62,4 +69,18 @@ type KubeInKubeConfig struct {
 	//
 	//// Registry contains the configuration for the registry in kubernetes cluster.
 	//Registry RegistryConfig `yaml:"registry" json:"registry,omitempty"`
+
+	//TenantEntrypoint TenantEntrypoint `yaml:"tenantEntrypoint" json:"tenantEntrypoint,omitempty"`
+	// +optional
+	TenantEntrypoint TenantEntrypoint `yaml:"tenantEntrypoint" json:"tenantEntrypoint,omitempty"`
+}
+
+// TenantEntrypoint contains the configuration for the tenant entrypoint.
+type TenantEntrypoint struct {
+	// ExternalIP is the external ip of the tenant entrypoint.
+	// +optional
+	ExternalIps []string `json:"externalIps,omitempty"`
+	// ExternalVips is the external vips of the tenant entrypoint.
+	// +optional
+	ExternalVips []string `json:"externalVips,omitempty"`
 }

pkg/apis/kosmos/v1alpha1/virtualcluster_types.go

@@ -56,6 +56,9 @@ type VirtualClusterSpec struct {
 	// +optional
 	ExternalIps []string `json:"externalIps,omitempty"`
 
+	// KubeInKubeConfig is the external config of virtual cluster
+	// +optional
+	KubeInKubeConfig KubeInKubeConfig `json:"kubeInKubeConfig,omitempty"`
 	// PromotePolicies definites the policies for promote to the kubernetes's control plane
 	// +required
 	PromotePolicies []PromotePolicy `json:"promotePolicies,omitempty"`
@@ -139,6 +142,8 @@ type VirtualClusterStatus struct {
 	Port int32 `json:"port,omitempty"`
 	// +optional
 	PortMap map[string]int32 `json:"portMap,omitempty"`
+	// +optional
+	VipMap map[string]string `json:"vipMap,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object