Skip to content

Commit

Permalink
Merge pull request #248 from OrangeBao/main
Browse files Browse the repository at this point in the history
feat: support exec and logs
  • Loading branch information
kosmos-robot committed Nov 17, 2023
2 parents fd4ecd0 + ef03f68 commit f3ba4e2
Show file tree
Hide file tree
Showing 31 changed files with 4,611 additions and 104 deletions.
11 changes: 11 additions & 0 deletions cmd/clustertree/cluster-manager/app/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
podcontrollers "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers/pod"
"github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers/pv"
"github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/controllers/pvc"
nodeserver "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/node-server"
leafUtils "github.com/kosmos.io/kosmos/pkg/clustertree/cluster-manager/utils"
"github.com/kosmos.io/kosmos/pkg/scheme"
"github.com/kosmos.io/kosmos/pkg/sharedcli/klogflag"
Expand Down Expand Up @@ -259,6 +260,16 @@ func run(ctx context.Context, opts *options.Options) error {
}
}()

nodeServer := nodeserver.NodeServer{
RootClient: mgr.GetClient(),
GlobalLeafManager: globalleafManager,
}
go func() {
if err := nodeServer.Start(ctx, opts); err != nil {
klog.Errorf("failed to start node server: %v", err)
}
}()

rootResourceManager.InformerFactory.Start(ctx.Done())
rootResourceManager.KosmosInformerFactory.Start(ctx.Done())
if !cache.WaitForCacheSync(ctx.Done(), rootResourceManager.EndpointSliceInformer.HasSynced, rootResourceManager.ServiceInformer.HasSynced) {
Expand Down
28 changes: 28 additions & 0 deletions deploy/clustertree-cluster-manager.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,17 @@ subjects:
name: clustertree
namespace: kosmos-system
---
apiVersion: v1
kind: Secret
metadata:
name: clustertree-cluster-manager
namespace: kosmos-system
type: Opaque
data:
cert.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQzakNDQXNhZ0F3SUJBZ0lJVWE0NWVxZmI0c0V3RFFZSktvWklodmNOQVFFTEJRQXdmekVMTUFrR0ExVUUKQmhNQ1ZWTXhEekFOQmdOVkJBZ1RCazl5WldkdmJqRVJNQThHQTFVRUJ4TUlVRzl5ZEd4aGJtUXhHREFXQmdOVgpCQW9URDNacmRXSmxiR1YwTFcxdlkyc3RNREVZTUJZR0ExVUVDeE1QZG10MVltVnNaWFF0Ylc5amF5MHdNUmd3CkZnWURWUVFERXc5MmEzVmlaV3hsZEMxdGIyTnJMVEF3SGhjTk1UZ3hNVEkyTVRJd016SXpXaGNOTVRrd01qSTEKTVRnd09ESXpXakIvTVFzd0NRWURWUVFHRXdKVlV6RVBNQTBHQTFVRUNCTUdUM0psWjI5dU1SRXdEd1lEVlFRSApFd2hRYjNKMGJHRnVaREVZTUJZR0ExVUVDaE1QZG10MVltVnNaWFF0Ylc5amF5MHdNUmd3RmdZRFZRUUxFdzkyCmEzVmlaV3hsZEMxdGIyTnJMVEF4R0RBV0JnTlZCQU1URDNacmRXSmxiR1YwTFcxdlkyc3RNRENDQVNJd0RRWUoKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTHJ5SHZLM1VCQkJxR1YyRnB3eW1mMHAvWUtHUUE5cgpOdTBONmYyK1JrVVhMdVFYRytXZEZRbDNaUXliUExmQ0UyaHdGY2wzSUYrM2hDelkzLzJVSXlHQmxvQklmdDdLCllGTE0zWVdKRHk1RWxLRGcxYk5EU0x6RjZ0a3BOTERuVmxna1BQSVR6cEVISUF1K0JUNURaR1doWUFXTy9EaXIKWGR4b0pCT2hQWlpDY0JDVitrd1FRUGJzWHpaeStxN1FoeDI3MENSTUlYc285QzVMSmhHWUw5ZndzeG11a0FPUgo1NlNtZnNBYW1sN1VPbHpISVRSRHdENUFRMUJrVFNFRnkwOGRrNkpBWUw4TERMaGdhTG9Xb1YwR2UyZ09JZXBSCmpwbDg3ZEdiU1ZHeUJIbVRYdjRvNnV0cVQ2UzZuVTc2TG45TlNpN1loTXFqOHVXdjBwVERsWWNDQXdFQUFhTmUKTUZ3d0RnWURWUjBQQVFIL0JBUURBZ1dnTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjRApBakFNQmdOVkhSTUJBZjhFQWpBQU1CMEdBMVVkRGdRV0JCUVZId1Uxc3k3UW53MVd2VnZGTGNacmhvVDQwREFOCkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQXNOR05LejFKd2Z3ZzdyWWFPN1ZGL3phbjAxWFhaRlAxYm5GWW5YSnUKMTVSemhPQk1zcDNLdldDVmh3VWZ4TmU4R2hVRFN4MnRtUzVFQS84b2FFbmdMRmwzanRSM3BuVU5Pd0RWbHpseQpRT0NOM3JsT2k0K3AyNkx2TWlBRnA1aHhYQXYzTE9SczZEenI2aDMvUVR0bFY1akRTaFVPWFpkRmRPUEpkWjJuCmc0Ymlyckc3TU82dnd2UjhDaU5jUTI2YitiOHA5QkdYYkU4YnNKb0htY3NxeWE4ZmJWczJuNkNkRUplSSs0aEQKTjZ4bG81U3Zoakg1dEZJSTdlQ1ZlZHlaR2wwQkt2a29jT2lnTGdxOFgrSnpGeGoxd3RkbXRYdjdzamRLY0I5cgo2VFdHSlJyWlZ4b3hVT3paaHB4VWozai9wTGFSY0RtdHRTSkN1RHUzTkF0a2dRPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
key.pem: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdXZJZThyZFFFRUdvWlhZV25ES1ovU245Z29aQUQyczI3UTNwL2I1R1JSY3U1QmNiCjVaMFZDWGRsREpzOHQ4SVRhSEFWeVhjZ1g3ZUVMTmpmL1pRaklZR1dnRWgrM3NwZ1VzemRoWWtQTGtTVW9PRFYKczBOSXZNWHEyU2swc09kV1dDUTg4aFBPa1FjZ0M3NEZQa05rWmFGZ0JZNzhPS3RkM0dna0U2RTlsa0p3RUpYNgpUQkJBOXV4Zk5uTDZydENISGJ2UUpFd2hleWowTGtzbUVaZ3YxL0N6R2E2UUE1SG5wS1ord0JxYVh0UTZYTWNoCk5FUEFQa0JEVUdSTklRWExUeDJUb2tCZ3Z3c011R0JvdWhhaFhRWjdhQTRoNmxHT21YenQwWnRKVWJJRWVaTmUKL2lqcTYycFBwTHFkVHZvdWYwMUtMdGlFeXFQeTVhL1NsTU9WaHdJREFRQUJBb0lCQUVOODR0VkdmaDNRUmlXUwpzdWppajVySVROK1E3WkZqYUNtOTZ5b1NSYlh0ZjUwU0JwMG16eEJpek5UM09iMHd6K2JWQjloNksvTENBbkphClBNcURid2RLaS9WMXRtOWhhZEthYUtJcmI1S0phWXFHZ0Q4OTNBVmlBYjB4MWZiREhQV201MldRNXZLT092QmkKUWV4UFVmQXFpTXFZNnM3ZWRuejZENFFTb25RYW14Q1VQQlBZdnVkbWF5SHRQbGM4UWI2ZVkwVitwY2RGblcwOApTRFpYWU94ZXkzL0lBalp5ZGNBN1hndk5TYys2WE93bWhLc0dBVzcxdUZUVGFnSnZ6WDNlUENZMTRya0dKbURHCm0vMTBob1c2Tk1LR2VWL1J5WDNkWDBqSm1EazFWZnhBUVczeHBPaXBaZmdmdmdhdkNPcUhuS0E2SThkSzN6aGcKdkU5QmxlRUNnWUVBODdYL3p0UVpESTRxT1RBOUNXL25NWGZ3QXk5UU8xSzZiR2hCSFV1N0pzNHBxZ3h1SDhGawpoUWdRSzdWOGlhc255L2RDeWo2Q3UzUUpOb2Z4dWRBdkxMUUtrcXV5UU9hK3pxRkNVcFZpZDdKVlJNY1JMSmx0CjNIbHlDTnZWbGhmakRUMGNJMlJkVTQ1cThNblpveTFmM0RQWkIxNmNIYjNITDl6MWdRWlRpWEVDZ1lFQXhGOWEKNjhTYnhtV0ZCSzdQYW9iSTh3VmZEb1Rpckhtb0F2bnlwWUswb1FrQVg4Vm1FbXRFRXMyK04xbW9LalNUUHIrdAp1czRKS2d1QTh6MnR1TGs1aitlRit6RGwvMlUrN2RqVEY4RkNOcHJ3ejNzWHI0MjdHQ0lHTDVZdnBJQlorVEw4CkJqaTJ1eW9vOGs5U0FXTWI0T2JPemZHbTR0ZUN2Y2lTOTlndzBuY0NnWUF0NUdiQVZ0WkVzL3lsZWp6MEt2dFoKS0dHczU5cnU0TncwRDhtN0w0aVZmUnNCWjRmUk9RU3B2R1AzSnh6RmU5SnBxUzBOa29uaHJLOFRjclFGTG52RApxaitYY1BlSEd5eHhFcEsvcEZ1L2VIaHdGQ0JheXFXU2I5Z1diUGNpWldzZkVoUGJZa25rc3h2V0xkeHF5dCtUClFyd3FsQmxIekhYV3dJQUdoTjkwTVFLQmdRQzVDWWtwQkZnc3VGaUJNeCtySjFxTzlJNi9wYVBhRmNDbEhWVHgKZEpvejY4RjRmUTlUWjlQN1MvZGpQSTVqUnF0QXcyazJ6eEovbGR0cVdNSXJnQTJuZGVnZjY5R3R1SDkxcTR3dApwQ042Uk1HSklGb1BTQ1AxOTRtUXFabzNEZUs2R0xxMk9oYWxnbktXOFBzNjUyTExwM0ZUU2RPUmlMVmZrM0k1CkxIUEV2UUtCZ0RDeGEvM3ZuZUc4dmdzOEFyRWpOODlCL1l4TzFxSVU1bXhKZTZaYWZiODFOZGhZVWpmUkFWcm8KQUxUb2ZpQXBNc25EYkpESE1pd3Z3Y0RVSGJQTHBydUs4MFIvL3ptWDdYZW4rRis1b2JmU1E4ajBHU21tZVdGUQpTVkc2QXBOdGt0TFBJMG5LMm5FSUgvUXg0b3VHQzlOMHBBRFJDbFFRUFN4RVBtRHZmNHhmCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCgo=

---
apiVersion: apps/v1
kind: Deployment
metadata:
Expand All @@ -50,7 +61,24 @@ spec:
- name: clustertree-cluster-manager
image: ghcr.io/kosmos-io/clustertree-cluster-manager:__VERSION__
imagePullPolicy: Always
env:
- name: APISERVER_CERT_LOCATION
value: /etc/cluster-tree/cert/cert.pem
- name: APISERVER_KEY_LOCATION
value: /etc/cluster-tree/cert/key.pem
- name: KNODE_POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: credentials
mountPath: "/etc/cluster-tree/cert"
readOnly: true
command:
- clustertree-cluster-manager
- --multi-cluster-service=true
- --v=4
volumes:
- name: credentials
secret:
secretName: clustertree-cluster-manager
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
github.com/go-logr/logr v1.2.3
github.com/gogo/protobuf v1.3.2
github.com/google/go-cmp v0.5.9
github.com/gorilla/mux v1.8.1
github.com/olekukonko/tablewriter v0.0.4
github.com/onsi/ginkgo/v2 v2.9.2
github.com/onsi/gomega v1.27.4
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,8 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK
github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU=
github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
Expand Down
61 changes: 14 additions & 47 deletions pkg/clustertree/cluster-manager/cluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ func (c *ClusterController) Reconcile(ctx context.Context, request reconcile.Req
c.ManagerCancelFuncs[cluster.Name] = &cancel
c.ControllerManagersLock.Unlock()

if err = c.setupControllers(mgr, cluster, nodes, leafDynamic, leafClient, kosmosClient); err != nil {
if err = c.setupControllers(mgr, cluster, nodes, leafDynamic, leafClient, kosmosClient, config); err != nil {
return reconcile.Result{}, fmt.Errorf("failed to setup cluster %s controllers: %v", cluster.Name, err)
}

Expand Down Expand Up @@ -235,8 +235,17 @@ func (c *ClusterController) clearClusterControllers(cluster *kosmosv1alpha1.Clus
c.GlobalLeafManager.RemoveLeafResource(cluster.Name)
}

func (c *ClusterController) setupControllers(mgr manager.Manager, cluster *kosmosv1alpha1.Cluster, nodes []*corev1.Node, clientDynamic *dynamic.DynamicClient, leafClientset kubernetes.Interface, kosmosClient kosmosversioned.Interface) error {
leafResource := &leafUtils.LeafResource{
func (c *ClusterController) setupControllers(mgr manager.Manager, cluster *kosmosv1alpha1.Cluster, nodes []*corev1.Node, clientDynamic *dynamic.DynamicClient, leafClientset kubernetes.Interface, kosmosClient kosmosversioned.Interface, leafRestConfig *rest.Config) error {
isNode2NodeFunc := func(cluster *kosmosv1alpha1.Cluster) bool {
return cluster.Spec.ClusterTreeOptions.LeafModels != nil
}

clusterName := fmt.Sprintf("%s%s", utils.KosmosNodePrefix, cluster.Name)
if isNode2NodeFunc(cluster) {
clusterName = cluster.Name
}

c.GlobalLeafManager.AddLeafResource(clusterName, &leafUtils.LeafResource{
Client: mgr.GetClient(),
DynamicClient: clientDynamic,
Clientset: leafClientset,
Expand All @@ -246,9 +255,8 @@ func (c *ClusterController) setupControllers(mgr manager.Manager, cluster *kosmo
Namespace: "",
IgnoreLabels: strings.Split("", ","),
EnableServiceAccount: true,
}

c.GlobalLeafManager.AddLeafResource(cluster.Name, leafResource, cluster.Spec.ClusterTreeOptions.LeafModels, nodes)
RestConfig: leafRestConfig,
}, cluster.Spec.ClusterTreeOptions.LeafModels, nodes)

nodeResourcesController := controllers.NodeResourcesController{
Leaf: mgr.GetClient(),
Expand Down Expand Up @@ -325,47 +333,6 @@ func (c *ClusterController) setupStorageControllers(mgr manager.Manager, isOne2O
return nil
}

// nolint
func (c *ClusterController) setNodeStatus(ctx context.Context, nodeName string, leafClient kubernetes.Interface, node *corev1.Node, isNode2Node bool) error {
if isNode2Node {
if leafnode, err := leafClient.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{}); err != nil {
klog.Errorf("create node %s failed, cannot get node from leaf cluster, err: %v", nodeName, err)
return err
} else {
node.Status = leafnode.Status
address, err := leafUtils.SortAddress(ctx, c.RootClientset, nodeName, leafClient, node.Status.Addresses)
if err != nil {
return err
}
node.Status.Addresses = address
return nil
}
}

leafnodes, err := leafClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{
// TODO: LabelSelector
})
if err != nil {
klog.Errorf("create node %s failed, cannot get node from leaf cluster, err: %v", nodeName, err)
return err
}

if len(leafnodes.Items) == 0 {
klog.Errorf("create node %s failed, cannot get node from leaf cluster, len of leafnodes is 0", nodeName)
return err
}

address, err := leafUtils.SortAddress(ctx, c.RootClientset, nodeName, leafClient, leafnodes.Items[0].Status.Addresses)

if err != nil {
return err
}

node.Status.Addresses = address

return nil
}

func (c *ClusterController) createNode(ctx context.Context, cluster *kosmosv1alpha1.Cluster, leafClient kubernetes.Interface) ([]*corev1.Node, error) {
serverVersion, err := leafClient.Discovery().ServerVersion()
if err != nil {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ func (c *NodeResourcesController) Reconcile(ctx context.Context, request reconci
clone.Spec = node.Spec
clone.Spec.Taints = rootNode.Spec.Taints
clone.Status = node.Status
clone.Status.Addresses = leafUtils.GetAddress()
}
}

Expand Down
73 changes: 73 additions & 0 deletions pkg/clustertree/cluster-manager/node-server/api/errdefs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package api

import (
"errors"
"fmt"
)

const (
ERR_NOT_FOUND = "ErrNotFound"
ERR_INVALID_INPUT = "ErrInvalidInput"
)

type causal interface {
Cause() error
error
}

type ErrNodeServer interface {
GetErrorType() string
error
}

type errNodeServer struct {
errType string
error
}

func (e *errNodeServer) GetErrorType() string {
return e.errType
}

func ErrNotFound(msg string) error {
return &errNodeServer{ERR_NOT_FOUND, errors.New(msg)}
}

func ErrInvalidInput(msg string) error {
return &errNodeServer{ERR_INVALID_INPUT, errors.New(msg)}
}

func IsMatchErrType(err error, errType string) bool {
if err == nil {
return false
}
if e, ok := err.(ErrNodeServer); ok {
return e.GetErrorType() == errType
}

if e, ok := err.(causal); ok {
return IsMatchErrType(e.Cause(), errType)
}

return false
}

func IsNotFound(err error) bool {
return IsMatchErrType(err, ERR_NOT_FOUND)
}

func IsInvalidInput(err error) bool {
return IsMatchErrType(err, ERR_INVALID_INPUT)
}

func ConvertNotFound(err error) error {
return &errNodeServer{ERR_NOT_FOUND, err}
}

func ConvertInvalidInput(err error) error {
return &errNodeServer{ERR_INVALID_INPUT, err}
}

func ErrInvalidInputf(format string, args ...interface{}) error {
return &errNodeServer{ERR_INVALID_INPUT, fmt.Errorf(format, args...)}
}
Loading

0 comments on commit f3ba4e2

Please sign in to comment.