Merge pull request #578 from zhouhaoA1/feature_components_deploy

add install scheduler component feature for kosmosctl
kosmos-io · May 23, 2024 · ad3329d · ad3329d
2 parents db6b995 + c79cd06
commit ad3329d
Show file tree

Hide file tree

Showing 8 changed files with 502 additions and 0 deletions.
diff --git a/pkg/kosmosctl/install/install.go b/pkg/kosmosctl/install/install.go
@@ -40,6 +40,9 @@ var installExample = templates.Examples(i18n.T(`
 		
 		# Install clusterlink module to Kosmos control plane and set the necessary parameters, e.g: 
 		kosmosctl install -m clusterlink --cni cni-name --default-nic nic-name
+
+		# Install kosmos-scheduler to Kosmos control plane, e.g: 
+		kosmosctl install -m scheduler
 		
 		# Install coredns module to Kosmos control plane, e.g: 
 		kosmosctl install -m coredns`))
@@ -199,6 +202,16 @@ func (o *CommandInstallOptions) Run() error {
 			return err
 		}
 		util.CheckInstall("Clustertree")
+	case utils.Scheduler:
+		err := o.runScheduler()
+		if err != nil {
+			return err
+		}
+		err = o.createControlCluster()
+		if err != nil {
+			return err
+		}
+		util.CheckInstall("Scheduler")
 	case utils.All:
 		err := o.runClusterlink()
 		if err != nil {
@@ -503,6 +516,122 @@ func (o *CommandInstallOptions) runClustertree() error {
 	return nil
 }
 
+func (o *CommandInstallOptions) runScheduler() error {
+	klog.Info("Start creating kosmos-scheduler...")
+	namespace := &corev1.Namespace{}
+	namespace.Name = o.Namespace
+	_, err := o.K8sClient.CoreV1().Namespaces().Create(context.TODO(), namespace, metav1.CreateOptions{})
+	if err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return fmt.Errorf("kosmosctl install clustertree run error, namespace options failed: %v", err)
+		}
+	}
+	klog.Info("Namespace " + o.Namespace + " has been created.")
+
+	klog.Info("Start creating kosmos-scheduler ServiceAccount...")
+	schedulerSA, err := util.GenerateServiceAccount(manifest.SchedulerServiceAccount, manifest.ServiceAccountReplace{
+		Namespace: o.Namespace,
+	})
+	if err != nil {
+		return err
+	}
+	_, err = o.K8sClient.CoreV1().ServiceAccounts(o.Namespace).Create(context.TODO(), schedulerSA, metav1.CreateOptions{})
+	if err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return fmt.Errorf("kosmosctl install scheduler run error, serviceaccount options failed: %v", err)
+		}
+	}
+	klog.Info("ServiceAccount " + schedulerSA.Name + " has been created.")
+
+	klog.Info("Start creating kosmos-scheduler ClusterRole...")
+	schedulerCR, err := util.GenerateClusterRole(manifest.SchedulerClusterRole, nil)
+	if err != nil {
+		return err
+	}
+	_, err = o.K8sClient.RbacV1().ClusterRoles().Create(context.TODO(), schedulerCR, metav1.CreateOptions{})
+	if err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return fmt.Errorf("kosmosctl install scheduler run error, clusterrole options failed: %v", err)
+		}
+	}
+	klog.Info("ClusterRole " + schedulerCR.Name + " has been created.")
+
+	klog.Info("Start creating kosmos-scheduler ClusterRoleBinding...")
+	schedulerCRB, err := util.GenerateClusterRoleBinding(manifest.SchedulerClusterRoleBinding, manifest.ClusterRoleBindingReplace{
+		Namespace: o.Namespace,
+	})
+	if err != nil {
+		return err
+	}
+	_, err = o.K8sClient.RbacV1().ClusterRoleBindings().Create(context.TODO(), schedulerCRB, metav1.CreateOptions{})
+	if err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return fmt.Errorf("kosmosctl install scheduler run error, clusterrolebinding options failed: %v", err)
+		}
+	}
+	klog.Info("ClusterRoleBinding " + schedulerCRB.Name + " has been created.")
+
+	klog.Info("Start creating kosmos-scheduler ConfigMap...")
+	scheduleConfigFile, err := util.GenerateConfigMap(manifest.SchedulerConfigmap, manifest.ConfigmapReplace{
+		Namespace: o.Namespace,
+	})
+	if err != nil {
+		return err
+	}
+	_, err = o.K8sClient.CoreV1().ConfigMaps(o.Namespace).Create(context.TODO(), scheduleConfigFile, metav1.CreateOptions{})
+	if err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return fmt.Errorf("kosmosctl install scheduler run error, configmap options failed: %v", err)
+		}
+	}
+	klog.Infof("ConfigMap %s has been created.", scheduleConfigFile.Name)
+
+	hostkubeConfigMap := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      utils.HostKubeConfigName,
+			Namespace: o.Namespace,
+		},
+		Data: map[string]string{
+			"kubeconfig": string(func() []byte {
+				if len(o.InnerKubeConfigStream) != 0 {
+					return o.InnerKubeConfigStream
+				}
+				return o.HostKubeConfigStream
+			}()),
+		},
+	}
+	_, err = o.K8sClient.CoreV1().ConfigMaps(o.Namespace).Create(context.TODO(), hostkubeConfigMap, metav1.CreateOptions{})
+	if err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return fmt.Errorf("kosmosctl install scheduler run error, configmap options failed: %v", err)
+		}
+	}
+	klog.Info("ConfigMap host-kubeconfig has been created.")
+
+	klog.Info("Start creating kosmos-scheduler Deployment...")
+	schedulerDeploy, err := util.GenerateDeployment(manifest.SchedulerDeployment, manifest.DeploymentReplace{
+		Namespace:       o.Namespace,
+		ImageRepository: o.ImageRegistry,
+		Version:         o.Version,
+	})
+	if err != nil {
+		return err
+	}
+	_, err = o.K8sClient.AppsV1().Deployments(o.Namespace).Create(context.Background(), schedulerDeploy, metav1.CreateOptions{})
+	if err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return fmt.Errorf("kosmosctl install clustertree run error, deployment options failed: %v", err)
+		}
+	}
+	label := map[string]string{"app": schedulerDeploy.Labels["app"]}
+	if err = util.WaitPodReady(o.K8sClient, schedulerDeploy.Namespace, util.MapToString(label), o.WaitTime); err != nil {
+		return fmt.Errorf("kosmosctl install scheduler run error, deployment options failed: %v", err)
+	} else {
+		klog.Info("Deployment kosmos-scheduler has been created.")
+	}
+	return nil
+}
+
 func (o *CommandInstallOptions) createOperator() error {
 	klog.Info("Start creating ClusterLink-Operator...")
 	operatorDeploy, err := util.GenerateDeployment(manifest.KosmosOperatorDeployment, manifest.DeploymentReplace{

diff --git a/pkg/kosmosctl/manifest/manifest_clusterrolebindings.go b/pkg/kosmosctl/manifest/manifest_clusterrolebindings.go
@@ -82,6 +82,21 @@ subjects:
     name: coredns
     namespace: {{ .Namespace }}
 `
+
+	SchedulerClusterRoleBinding = `
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kosmos-scheduler
+subjects:
+  - kind: ServiceAccount
+    name: kosmos-scheduler
+    namespace: {{ .Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kosmos-scheduler
+`
 )
 
 type ClusterRoleBindingReplace struct {

diff --git a/pkg/kosmosctl/manifest/manifest_clusterroles.go b/pkg/kosmosctl/manifest/manifest_clusterroles.go
@@ -65,4 +65,161 @@ rules:
   - nonResourceURLs: ['*']
     verbs: ["get"]
 `
+
+	SchedulerClusterRole = `
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kosmos-scheduler
+rules:
+  - verbs:
+      - get
+      - list
+      - watch
+    apiGroups:
+      - kosmos.io
+    resources:
+      - '*'
+  - verbs:
+      - create
+      - patch
+      - update
+    apiGroups:
+      - ''
+      - events.k8s.io
+    resources:
+      - events
+  - verbs:
+      - create
+    apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+  - verbs:
+      - get
+      - update
+    apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    resourceNames:
+      - kosmos-scheduler
+  - verbs:
+      - create
+    apiGroups:
+      - ''
+    resources:
+      - endpoints
+  - verbs:
+      - get
+      - update
+    apiGroups:
+      - ''
+    resources:
+      - endpoints
+  - verbs:
+      - get
+      - list
+      - watch
+    apiGroups:
+      - ''
+    resources:
+      - nodes
+  - verbs:
+      - delete
+      - get
+      - list
+      - watch
+    apiGroups:
+      - ''
+    resources:
+      - pods
+  - verbs:
+      - create
+    apiGroups:
+      - ''
+    resources:
+      - bindings
+      - pods/binding
+  - verbs:
+      - patch
+      - update
+    apiGroups:
+      - ''
+    resources:
+      - pods/status
+  - verbs:
+      - get
+      - list
+      - watch
+    apiGroups:
+      - ''
+    resources:
+      - replicationcontrollers
+      - services
+  - verbs:
+      - get
+      - list
+      - watch
+    apiGroups:
+      - apps
+      - extensions
+    resources:
+      - replicasets
+  - verbs:
+      - get
+      - list
+      - watch
+    apiGroups:
+      - apps
+    resources:
+      - statefulsets
+  - verbs:
+      - get
+      - list
+      - watch
+    apiGroups:
+      - policy
+    resources:
+      - poddisruptionbudgets
+  - verbs:
+      - get
+      - list
+      - watch
+      - update
+    apiGroups:
+      - ''
+    resources:
+      - persistentvolumeclaims
+      - persistentvolumes
+  - verbs:
+      - create
+    apiGroups:
+      - authentication.k8s.io
+    resources:
+      - tokenreviews
+  - verbs:
+      - create
+    apiGroups:
+      - authorization.k8s.io
+    resources:
+      - subjectaccessreviews
+  - verbs:
+      - get
+      - list
+      - watch
+    apiGroups:
+      - storage.k8s.io
+    resources:
+      - '*'
+  - verbs:
+      - get
+      - list
+      - watch
+    apiGroups:
+      - ''
+    resources:
+      - configmaps
+      - namespaces
+`
 )
diff --git a/pkg/kosmosctl/manifest/manifest_configmaps.go b/pkg/kosmosctl/manifest/manifest_configmaps.go
@@ -40,6 +40,59 @@ metadata:
   name: coredns-customer-hosts
   namespace: {{ .Namespace }}
 `
+
+	SchedulerConfigmap = `
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: scheduler-config
+  namespace: {{ .Namespace }}
+data:
+  scheduler-config.yaml: |
+    apiVersion: kubescheduler.config.k8s.io/v1beta1
+    kind: KubeSchedulerConfiguration
+    leaderElection:
+      leaderElect: true
+      resourceName: kosmos-scheduler
+      resourceNamespace: {{ .Namespace }}
+    profiles:
+      - schedulerName: default-scheduler
+        plugins:
+          preFilter:
+            disabled:
+              - name: "VolumeBinding"
+            enabled:
+              - name: "LeafNodeVolumeBinding"
+          filter:
+            disabled:
+              - name: "VolumeBinding"
+              - name: "TaintToleration"
+            enabled:
+              - name: "LeafNodeTaintToleration"
+              - name: "LeafNodeVolumeBinding"
+              - name: "LeafNodeDistribution"
+          score:
+            disabled:
+              - name: "VolumeBinding"
+          reserve:
+            disabled:
+              - name: "VolumeBinding"
+            enabled:
+              - name: "LeafNodeVolumeBinding"
+          preBind:
+            disabled:
+              - name: "VolumeBinding"
+            enabled:
+              - name: "LeafNodeVolumeBinding"
+        pluginConfig:
+          - name: LeafNodeVolumeBinding
+            args:
+              bindTimeoutSeconds: 5
+          - name: LeafNodeDistribution
+            args:
+            # kubeConfigPath: "REPLACE_ME_WITH_KUBE_CONFIG_PATH"
+              kubeConfigPath: "/etc/kubernetes/kubeconfig"
+`
 )
 
 type ConfigmapReplace struct {