Merge pull request #684 from OrangeBao/main

Merge release0.4.0 to main

.gitignore

@@ -31,7 +31,6 @@ kube-config
 __debug_bin*
 
 ignore_dir
-
 cmd/kubenest/node-agent/app.log
 cmd/kubenest/node-agent/cert.pem
 cmd/kubenest/node-agent/key.pem

deploy/crds/kosmos.io_kubenestconfigurations.yaml

@@ -0,0 +1,63 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.11.0
+  creationTimestamp: null
+  name: kubenestconfigurations.kosmos.io
+spec:
+  group: kosmos.io
+  names:
+    kind: KubeNestConfiguration
+    listKind: KubeNestConfigurationList
+    plural: kubenestconfigurations
+    singular: kubenestconfiguration
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: KubeNestConfiguration defines the configuration for KubeNest
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          kosmosKubeConfig:
+            properties:
+              allowNodeOwnbyMulticluster:
+                description: AllowNodeOwnbyMulticluster indicates whether to allow
+                  nodes to be owned by multiple clusters.
+                type: boolean
+            type: object
+          kubeInKubeConfig:
+            properties:
+              admissionPlugins:
+                type: boolean
+              anpMode:
+                type: string
+              apiServerReplicas:
+                type: integer
+              clusterCIDR:
+                type: string
+              etcdStorageClass:
+                type: string
+              etcdUnitSize:
+                type: string
+              forceDestroy:
+                type: boolean
+            type: object
+          kubeNestType:
+            type: string
+          metadata:
+            type: object
+        type: object
+    served: true
+    storage: true

deploy/crds/kosmos.io_virtualclusters.yaml

@@ -47,6 +47,12 @@ spec:
                 description: ExternalIP is the external ip of the virtual kubernetes's
                   control plane
                 type: string
+              externalIps:
+                description: ExternalIps is the external ips of the virtual kubernetes's
+                  control plane
+                items:
+                  type: string
+                type: array
               kubeconfig:
                 description: Kubeconfig is the kubeconfig of the virtual kubernetes's
                   control plane
@@ -208,5 +214,4 @@ spec:
         type: object
     served: true
     storage: true
-    subresources:
-      status: {}
+    subresources: {}

deploy/virtual-cluster-host-port-cm.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kosmos-hostports
+  namespace: kosmos-system
+data:
+  config.yaml: |
+    # ports allocate for virtual cluster api server,from 33001, increment by 1 for each virtual cluster.Be careful not to use ports that are already in use
+    portsPool:
+      - 33001
+      - 33002
+      - 33003
+      - 33004
+      - 33005
+      - 33006
+      - 33007
+      - 33008
+      - 33009
+      - 33010
+    # when port is allocate from pool,it will be used for virtual cluster api server,and the port will be released after virtual cluster is deleted
+    clusterPorts:

deploy/virtual-cluster-operator.yml

@@ -271,3 +271,26 @@ data:
       - 33008
       - 33009
       - 33010
+---
+apiVersion: v1
+data:
+  egress_selector_configuration.yaml: |
+    apiVersion: apiserver.k8s.io/v1beta1
+    kind: EgressSelectorConfiguration
+    egressSelections:
+      - name: cluster
+        connection:
+          proxyProtocol: GRPC
+          transport:
+            uds:
+              udsName: /etc/kubernetes/konnectivity-server/konnectivity-server.socket
+      - name: master
+        connection:
+          proxyProtocol: Direct
+      - name: etcd
+        connection:
+          proxyProtocol: Direct
+kind: ConfigMap
+metadata:
+  name: kas-proxy-files
+  namespace: kas-proxy

pkg/apis/kosmos/v1alpha1/virtualcluster_types.go

@@ -27,11 +27,9 @@ const (
 
 // +genclient
 // +kubebuilder:resource:scope=Namespaced,shortName=vc
-// +kubebuilder:subresource:status
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +kubebuilder:printcolumn:name="STATUS",type=string,JSONPath=`.status.phase`
 // +kubebuilder:printcolumn:name="UPDATE-TIME",type=string,JSONPath=`.status.updateTime`
-
 type VirtualCluster struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`
@@ -54,6 +52,10 @@ type VirtualClusterSpec struct {
 	// +optional
 	ExternalIP string `json:"externalIP,omitempty"`
 
+	// ExternalIps is the external ips of the virtual kubernetes's control plane
+	// +optional
+	ExternalIps []string `json:"externalIps,omitempty"`
+
 	// PromotePolicies definites the policies for promote to the kubernetes's control plane
 	// +required
 	PromotePolicies []PromotePolicy `json:"promotePolicies,omitempty"`

pkg/kubenest/constants/constant.go

@@ -17,6 +17,7 @@ const (
 	DefaultImageRepositoryEnv        = "IMAGE_REPOSITIRY"
 	DefaultImageVersionEnv           = "IMAGE_VERSION"
 	DefaultCoreDnsImageTagEnv        = "COREDNS_IMAGE_TAG"
+	DefaultVirtualControllerLabelEnv = "VIRTUAL_CONTROLLER_LABEL"
 	VirtualClusterFinalizerName      = "kosmos.io/virtual-cluster-finalizer"
 	ServiceType                      = "NodePort"
 	EtcdServiceType                  = "ClusterIP"
@@ -26,7 +27,6 @@ const (
 	Label                            = "virtualCluster-app"
 	ComponentBeReadyTimeout          = 300 * time.Second
 	ComponentBeDeletedTimeout        = 300 * time.Second
-	DefauleVirtualControllerLabelEnv = "VIRTUAL_CONTROLLER_LABEL"
 
 	// CertificateBlockType is a possible value for pem.Block.Type.
 	CertificateBlockType           = "CERTIFICATE"

pkg/kubenest/controller/virtualcluster_init_controller.go

@@ -122,6 +122,7 @@ func (c *VirtualClusterInitController) Reconcile(ctx context.Context, request re
 			klog.Errorf("Error update virtualcluster %s status, err: %v", updatedCluster.Name, err)
 			return reconcile.Result{RequeueAfter: RequeueTime}, errors.Wrapf(err, "Error update virtualcluster %s status", updatedCluster.Name)
 		}
+
 		err = c.createVirtualCluster(updatedCluster, c.KubeNestOptions)
 		if err != nil {
 			klog.Errorf("Failed to create virtualcluster %s. err: %s", updatedCluster.Name, err.Error())
@@ -322,6 +323,10 @@ func (c *VirtualClusterInitController) assignWorkNodes(virtualCluster *v1alpha1.
 		return fmt.Errorf("list global nodes: %w", err)
 	}
 	allNodeInfos := make([]v1alpha1.NodeInfo, 0)
+	globalNodes := globalNodeList.Items
+	sort.Slice(globalNodes, func(i, j int) bool {
+		return globalNodes[i].Name < globalNodes[j].Name
+	})
 	for _, policy := range virtualCluster.Spec.PromotePolicies {
 		globalNodes, err := retrieveGlobalNodesWithLabelSelector(globalNodeList.Items, policy.LabelSelector)
 		if err != nil {