feat(registry): using etcd storage for chartmuseum (#1333)

cmd/tke-installer/app/installer/manifests/tke-registry-api/tke-registry-api.yaml

@@ -184,6 +184,13 @@ data:
     apiVersion: registry.config.tkestack.io/v1
     kind: RegistryConfiguration
     storage:
+      etcd:
+        cafile: "/app/certs/etcd-ca.crt"
+        certfile: "/app/certs/etcd.crt"
+        keyfile: "/app/certs/etcd.key"
+        endpoints:
+        - "https://etcd.kube-system:2379"
+        prefix: "/chart_backend_bucket"
       fileSystem:
         rootDirectory: /storage
       delete:

cmd/tke-installer/app/installer/manifests/tke-registry-controller/tke-registry-controller.yaml

@@ -106,6 +106,13 @@ data:
     apiVersion: registry.config.tkestack.io/v1
     kind: RegistryConfiguration
     storage:
+      etcd:
+        cafile: "/app/certs/etcd-ca.crt"
+        certfile: "/app/certs/etcd.crt"
+        keyfile: "/app/certs/etcd.key"
+        endpoints:
+        - "https://etcd.kube-system:2379"
+        prefix: "/chart_backend_bucket"
       fileSystem:
         rootDirectory: /storage
     security:

go.mod

@@ -3,8 +3,10 @@ module tkestack.io/tke
 go 1.12
 
 replace (
-	github.com/chartmuseum/storage => github.com/choujimmy/storage v0.0.0-20200507092433-6aea2df34764
+	github.com/chartmuseum/storage => github.com/choujimmy/storage v0.5.1-0.20210412121305-660c0e91489b
+	github.com/containerd/containerd => github.com/containerd/containerd v1.4.3
 	github.com/deislabs/oras => github.com/deislabs/oras v0.8.0
+	go.etcd.io/etcd => go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5
 	google.golang.org/grpc => google.golang.org/grpc v1.26.0
 	helm.sh/helm/v3 => helm.sh/helm/v3 v3.2.0
 	k8s.io/client-go => k8s.io/client-go v0.18.2
@@ -82,6 +84,7 @@ require (
 	gopkg.in/square/go-jose.v2 v2.4.1
 	gopkg.in/yaml.v2 v2.3.0
 	gotest.tools v2.2.0+incompatible
+	gotest.tools/v3 v3.0.3 // indirect
 	helm.sh/chartmuseum v0.12.0
 	helm.sh/helm/v3 v3.3.4
 	istio.io/api v0.0.0-20200715212100-dbf5277541ef

pkg/registry/apis/config/types.go

@@ -34,9 +34,9 @@ type RegistryConfiguration struct {
 	Redis         *Redis
 	DefaultTenant string
 	// +optional
-	DomainSuffix string
+	DomainSuffix  string
 	HarborEnabled bool
-	HarborCAFile string
+	HarborCAFile  string
 }
 
 type Storage struct {
@@ -47,8 +47,18 @@ type Storage struct {
 	// +optional
 	S3 *S3Storage
 	// +optional
+	Etcd *EtcdStorage
+	// +optional
 	Delete *Delete
 }
+type EtcdStorage struct {
+	CAFile    string
+	CertFile  string
+	KeyFile   string
+	EndPoints []string
+	// +optional
+	Prefix string
+}
 
 type FileSystemStorage struct {
 	RootDirectory string

pkg/registry/apis/config/v1/types.go

@@ -47,9 +47,20 @@ type Storage struct {
 	// +optional
 	S3 *S3Storage `json:"s3,omitempty" yaml:"s3,omitempty"`
 	// +optional
+	Etcd *EtcdStorage `json:"etcd,omitempty" yaml:"etcd,omitempty"`
+	// +optional
 	Delete *Delete `json:"delete,omitempty" yaml:"delete,omitempty"`
 }
 
+type EtcdStorage struct {
+	CAFile    string   `json:"cafile" yaml:"cafile"`
+	CertFile  string   `json:"certfile" yaml:"certfile"`
+	KeyFile   string   `json:"keyfile" yaml:"keyfile"`
+	EndPoints []string `json:"endpoints" yaml:"endpoints"`
+	// +optional
+	Prefix string `json:"prefix,omitempty" yaml:"prefix,omitempty"`
+}
+
 type FileSystemStorage struct {
 	RootDirectory string `json:"rootDirectory" yaml:"rootDirectory"`
 	// +optional

pkg/registry/apis/config/validation/validation.go

@@ -31,6 +31,24 @@ func ValidateRegistryConfiguration(rc *registryconfig.RegistryConfiguration) err
 	storageCount := 0
 	storageFld := field.NewPath("storage")
 
+	if rc.Storage.Etcd != nil {
+		storageCount++
+		subFld := storageFld.Child("etcd")
+
+		if rc.Storage.Etcd.CAFile == "" {
+			allErrors = append(allErrors, field.Required(subFld.Child("cafile"), "must be specify"))
+		}
+		if rc.Storage.Etcd.CertFile == "" {
+			allErrors = append(allErrors, field.Required(subFld.Child("certfile"), "must be specify"))
+		}
+		if rc.Storage.Etcd.KeyFile == "" {
+			allErrors = append(allErrors, field.Required(subFld.Child("keyfile"), "must be specify"))
+		}
+		if len(rc.Storage.Etcd.EndPoints) == 0 {
+			allErrors = append(allErrors, field.Required(subFld.Child("endpoints"), "must be specify"))
+		}
+	}
+
 	if rc.Storage.S3 != nil {
 		storageCount++
 		subFld := storageFld.Child("s3")

pkg/registry/chartmuseum/serveroptions/serveroptions.go

@@ -84,9 +84,14 @@ func buildStorageConfiguration(registryConfig *registryconfig.RegistryConfigurat
 	var backend storage.Backend
 	var err error
 	storageCfg := &registryConfig.Storage
-	if storageCfg.FileSystem != nil {
+	if storageCfg.Etcd != nil {
+		log.Info("Using etcd storage")
+		backend, err = buildETCDStorageConfiguration(registryConfig.Storage.Etcd)
+	} else if storageCfg.FileSystem != nil {
+		log.Info("Using filesystem storage")
 		backend = storage.Backend(storage.NewLocalFilesystemBackend(storageCfg.FileSystem.RootDirectory))
 	} else if storageCfg.S3 != nil {
+		log.Info("Using s3 storage")
 		backend, err = buildS3StorageConfiguration(registryConfig.Storage.S3)
 	}
 
@@ -100,6 +105,16 @@ func buildStorageConfiguration(registryConfig *registryconfig.RegistryConfigurat
 	return backend, nil
 }
 
+func buildETCDStorageConfiguration(cfg *registryconfig.EtcdStorage) (storage.Backend, error) {
+	return storage.NewEtcdCSBackend(
+		strings.Join(cfg.EndPoints, ","),
+		cfg.CAFile,
+		cfg.CertFile,
+		cfg.KeyFile,
+		cfg.Prefix,
+	), nil
+}
+
 func buildS3StorageConfiguration(cfg *registryconfig.S3Storage) (storage.Backend, error) {
 	awsConfig := aws.NewConfig()
 	sess, err := session.NewSession()