From 807227aaf0c4e83642abcda01f5f314648dcae0c Mon Sep 17 00:00:00 2001 From: Kuromesi Date: Sun, 9 Jun 2024 16:28:47 +0800 Subject: [PATCH] add support for credential provider plugin Signed-off-by: Kuromesi --- Makefile | 2 + cmd/daemon/main.go | 27 +++- pkg/daemon/criruntime/imageruntime/cri.go | 44 ++--- pkg/daemon/criruntime/imageruntime/docker.go | 36 ++--- .../imageruntime/fake_plugin/main.go | 152 ++++++++++++++++++ .../fake_plugin/plugin-config.yaml | 9 ++ .../criruntime/imageruntime/helpers_test.go | 21 +++ pkg/daemon/criruntime/imageruntime/pouch.go | 36 ++--- pkg/util/secret/parse.go | 12 +- 9 files changed, 277 insertions(+), 62 deletions(-) create mode 100644 pkg/daemon/criruntime/imageruntime/fake_plugin/main.go create mode 100644 pkg/daemon/criruntime/imageruntime/fake_plugin/plugin-config.yaml diff --git a/Makefile b/Makefile index b4d371610b..76f8e87bd9 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,9 @@ lint: golangci-lint ## Run golangci-lint against code. test: generate fmt vet manifests envtest ## Run tests echo $(ENVTEST) + go build -o pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin pkg/daemon/criruntime/imageruntime/fake_plugin/main.go && chmod +x pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test ./pkg/... -coverprofile cover.out + rm pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin coverage-report: ## Generate cover.html from cover.out go tool cover -html=cover.out -o cover.html diff --git a/cmd/daemon/main.go b/cmd/daemon/main.go index 4257a667e8..a11ee02e9b 100644 --- a/cmd/daemon/main.go +++ b/cmd/daemon/main.go @@ -17,6 +17,10 @@ limitations under the License. package main import ( + "os" + + "k8s.io/kubernetes/pkg/credentialprovider/plugin" + "flag" "math/rand" "net/http" @@ -34,12 +38,15 @@ import ( "github.com/openkruise/kruise/pkg/daemon" "github.com/openkruise/kruise/pkg/features" utilfeature "github.com/openkruise/kruise/pkg/util/feature" + "github.com/openkruise/kruise/pkg/util/secret" ) var ( - bindAddr = flag.String("addr", ":10221", "The address the metric endpoint and healthz binds to.") - pprofAddr = flag.String("pprof-addr", ":10222", "The address the pprof binds to.") - enablePprof = flag.Bool("enable-pprof", true, "Enable pprof for daemon.") + bindAddr = flag.String("addr", ":10221", "The address the metric endpoint and healthz binds to.") + pprofAddr = flag.String("pprof-addr", ":10222", "The address the pprof binds to.") + enablePprof = flag.Bool("enable-pprof", true, "Enable pprof for daemon.") + pluginConfigFile = flag.String("plugin-config-file", "/kruise/CredentialProviderPlugin.yaml", "The path of plugin config file.") + pluginBinDir = flag.String("plugin-bin-dir", "/kruise/plugins", "The path of directory of plugin binaries.") ) func main() { @@ -68,6 +75,20 @@ func main() { if err != nil { klog.Fatalf("Failed to new daemon: %v", err) } + + if _, err := os.Stat(*pluginConfigFile); err == nil { + err = plugin.RegisterCredentialProviderPlugins(*pluginConfigFile, *pluginBinDir) + if err != nil { + klog.Errorf("Failed to register credential provider plugins: %v", err) + } + } else if os.IsNotExist(err) { + klog.Infof("No plugin config file found, skipping: %s", *pluginConfigFile) + } else { + klog.Errorf("Failed to check plugin config file: %v", err) + } + // make sure the new docker key ring is made and set after the credential plugins are registered + secret.MakeAndSetKeyring() + if err := d.Run(ctx); err != nil { klog.Fatalf("Failed to start daemon: %v", err) } diff --git a/pkg/daemon/criruntime/imageruntime/cri.go b/pkg/daemon/criruntime/imageruntime/cri.go index 0f303a2606..957212d299 100644 --- a/pkg/daemon/criruntime/imageruntime/cri.go +++ b/pkg/daemon/criruntime/imageruntime/cri.go @@ -131,31 +131,31 @@ func (c *commonCRIImageService) pullImageV1(ctx context.Context, imageName, tag // for some runtime implementations. pullImageReq.SandboxConfig.Annotations[pullingImageSandboxConfigAnno] = "kruise-daemon" - if len(pullSecrets) > 0 { - var authInfos []daemonutil.AuthInfo - authInfos, err = secret.ConvertToRegistryAuths(pullSecrets, repoToPull) - if err == nil { - var pullErrs []error - for _, authInfo := range authInfos { - var pullErr error - klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username) - pullImageReq.Auth = &runtimeapi.AuthConfig{ - Username: authInfo.Username, - Password: authInfo.Password, - } - _, pullErr = c.criImageClient.PullImage(ctx, pullImageReq) - if pullErr == nil { - pipeW.CloseWithError(io.EOF) - return newImagePullStatusReader(pipeR), nil - } - klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr) - pullErrs = append(pullErrs, pullErr) - + var authInfos []daemonutil.AuthInfo + authInfos, err = secret.ConvertToRegistryAuths(pullSecrets, repoToPull) + if err == nil { + var pullErrs []error + for _, authInfo := range authInfos { + var pullErr error + klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username) + pullImageReq.Auth = &runtimeapi.AuthConfig{ + Username: authInfo.Username, + Password: authInfo.Password, } - if len(pullErrs) > 0 { - err = utilerrors.NewAggregate(pullErrs) + _, pullErr = c.criImageClient.PullImage(ctx, pullImageReq) + if pullErr == nil { + pipeW.CloseWithError(io.EOF) + return newImagePullStatusReader(pipeR), nil } + klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr) + pullErrs = append(pullErrs, pullErr) + + } + if len(pullErrs) > 0 { + err = utilerrors.NewAggregate(pullErrs) } + } else { + klog.Errorf("Failed to convert to auth info for registry, err %v", err) } // Try the default secret diff --git a/pkg/daemon/criruntime/imageruntime/docker.go b/pkg/daemon/criruntime/imageruntime/docker.go index 27892c4c51..1156881035 100644 --- a/pkg/daemon/criruntime/imageruntime/docker.go +++ b/pkg/daemon/criruntime/imageruntime/docker.go @@ -81,26 +81,26 @@ func (d *dockerImageService) PullImage(ctx context.Context, imageName, tag strin fullName := imageName + ":" + tag var ioReader io.ReadCloser - if len(pullSecrets) > 0 { - var authInfos []daemonutil.AuthInfo - authInfos, err = secret.ConvertToRegistryAuths(pullSecrets, registry) - if err == nil { - var pullErrs []error - for _, authInfo := range authInfos { - var pullErr error - klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username) - ioReader, pullErr = d.client.ImagePull(ctx, fullName, dockertypes.ImagePullOptions{RegistryAuth: authInfo.EncodeToString()}) - if pullErr == nil { - return newImagePullStatusReader(ioReader), nil - } - d.handleRuntimeError(pullErr) - klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr) - pullErrs = append(pullErrs, pullErr) - } - if len(pullErrs) > 0 { - err = utilerrors.NewAggregate(pullErrs) + var authInfos []daemonutil.AuthInfo + authInfos, err = secret.ConvertToRegistryAuths(pullSecrets, registry) + if err == nil { + var pullErrs []error + for _, authInfo := range authInfos { + var pullErr error + klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username) + ioReader, pullErr = d.client.ImagePull(ctx, fullName, dockertypes.ImagePullOptions{RegistryAuth: authInfo.EncodeToString()}) + if pullErr == nil { + return newImagePullStatusReader(ioReader), nil } + d.handleRuntimeError(pullErr) + klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr) + pullErrs = append(pullErrs, pullErr) + } + if len(pullErrs) > 0 { + err = utilerrors.NewAggregate(pullErrs) } + } else { + klog.Errorf("Failed to convert to auth info for registry, err %v", err) } // Try the default secret diff --git a/pkg/daemon/criruntime/imageruntime/fake_plugin/main.go b/pkg/daemon/criruntime/imageruntime/fake_plugin/main.go new file mode 100644 index 0000000000..e19814b3ce --- /dev/null +++ b/pkg/daemon/criruntime/imageruntime/fake_plugin/main.go @@ -0,0 +1,152 @@ +package main + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/runtime/serializer/json" + "k8s.io/component-base/logs" + "k8s.io/klog/v2" + "k8s.io/kubelet/pkg/apis/credentialprovider/install" + v1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1" +) + +var ( + scheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(scheme) +) + +func init() { + install.Install(scheme) +} + +func getCredentials(ctx context.Context, image string, args []string) (*v1.CredentialProviderResponse, error) { + response := &v1.CredentialProviderResponse{ + CacheKeyType: v1.RegistryPluginCacheKeyType, + Auth: map[string]v1.AuthConfig{ + "registry.plugin.com/test": { + Username: "user", + Password: "password", + }, + }, + } + return response, nil +} + +func runPlugin(ctx context.Context, r io.Reader, w io.Writer, args []string) error { + data, err := io.ReadAll(r) + if err != nil { + return err + } + + _, err = json.DefaultMetaFactory.Interpret(data) + if err != nil { + return err + } + + request, err := decodeRequest(data) + if err != nil { + return err + } + + if request.Image == "" { + return errors.New("image in plugin request was empty") + } + + // Deny all requests except for those where the image URL contains registry.plugin.com + // to test whether kruise could get expected auths if plugin fails to run + if !strings.Contains(request.Image, "registry.plugin.com") { + return errors.New("image in plugin request not supported: " + request.Image) + } + + response, err := getCredentials(ctx, request.Image, args) + if err != nil { + return err + } + + if response == nil { + return errors.New("CredentialProviderResponse from plugin was nil") + } + + encodedResponse, err := encodeResponse(response) + if err != nil { + return err + } + + writer := bufio.NewWriter(w) + defer writer.Flush() + if _, err := writer.Write(encodedResponse); err != nil { + return err + } + + return nil +} + +func decodeRequest(data []byte) (*v1.CredentialProviderRequest, error) { + obj, gvk, err := codecs.UniversalDecoder(v1.SchemeGroupVersion).Decode(data, nil, nil) + if err != nil { + return nil, err + } + + if gvk.Kind != "CredentialProviderRequest" { + return nil, fmt.Errorf("kind was %q, expected CredentialProviderRequest", gvk.Kind) + } + + if gvk.Group != v1.GroupName { + return nil, fmt.Errorf("group was %q, expected %s", gvk.Group, v1.GroupName) + } + + request, ok := obj.(*v1.CredentialProviderRequest) + if !ok { + return nil, fmt.Errorf("unable to convert %T to *CredentialProviderRequest", obj) + } + + return request, nil +} + +func encodeResponse(response *v1.CredentialProviderResponse) ([]byte, error) { + mediaType := "application/json" + info, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), mediaType) + if !ok { + return nil, fmt.Errorf("unsupported media type %q", mediaType) + } + + encoder := codecs.EncoderForVersion(info.Serializer, v1.SchemeGroupVersion) + data, err := runtime.Encode(encoder, response) + if err != nil { + return nil, fmt.Errorf("failed to encode response: %v", err) + } + + return data, nil +} + +func main() { + logs.InitLogs() + defer logs.FlushLogs() + + if err := newCredentialProviderCommand().Execute(); err != nil { + os.Exit(1) + } +} + +func newCredentialProviderCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "acr-credential-provider", + Short: "ACR credential provider for kubelet", + Run: func(cmd *cobra.Command, args []string) { + if err := runPlugin(context.TODO(), os.Stdin, os.Stdout, os.Args[1:]); err != nil { + klog.Errorf("Error running credential provider plugin: %v", err) + os.Exit(1) + } + }, + } + return cmd +} diff --git a/pkg/daemon/criruntime/imageruntime/fake_plugin/plugin-config.yaml b/pkg/daemon/criruntime/imageruntime/fake_plugin/plugin-config.yaml new file mode 100644 index 0000000000..d3ff288b21 --- /dev/null +++ b/pkg/daemon/criruntime/imageruntime/fake_plugin/plugin-config.yaml @@ -0,0 +1,9 @@ +apiVersion: kubelet.config.k8s.io/v1 +kind: CredentialProviderConfig +providers: + - name: fake-credential-plugin + matchImages: + - "registry.plugin.com" + - "registry.private.com" + defaultCacheDuration: "12h" + apiVersion: credentialprovider.kubelet.k8s.io/v1 \ No newline at end of file diff --git a/pkg/daemon/criruntime/imageruntime/helpers_test.go b/pkg/daemon/criruntime/imageruntime/helpers_test.go index a4a8039845..061cdeeb02 100644 --- a/pkg/daemon/criruntime/imageruntime/helpers_test.go +++ b/pkg/daemon/criruntime/imageruntime/helpers_test.go @@ -20,6 +20,8 @@ import ( "testing" v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/credentialprovider/plugin" "k8s.io/kubernetes/pkg/util/parsers" "github.com/openkruise/kruise/pkg/util/secret" @@ -103,7 +105,26 @@ func TestMatchRegistryAuths(t *testing.T) { }, ExpectMinValue: 0, }, + { + name: "test credential plugin if matched", + Image: "registry.plugin.com/test/echoserver:v1", + GetSecrets: func() []v1.Secret { + return []v1.Secret{} + }, + Expect: 1, + }, + } + pluginBinDir := "fake_plugin" + pluginConfigFile := "fake_plugin/plugin-config.yaml" + // credential plugin is configured for images with "registry.plugin.com" and "registry.private.com", + // however, only images with "registry.plugin.com" will return a fake credential, + // other images will be denied by the plugin and an error will be raised, + // this is to test whether kruise could get expected auths if plugin fails to run + err := plugin.RegisterCredentialProviderPlugins(pluginConfigFile, pluginBinDir) + if err != nil { + klog.Errorf("Failed to register credential provider plugins: %v", err) } + secret.MakeAndSetKeyring() for _, cs := range cases { t.Run(cs.name, func(t *testing.T) { repoToPull, _, _, err := parsers.ParseImageName(cs.Image) diff --git a/pkg/daemon/criruntime/imageruntime/pouch.go b/pkg/daemon/criruntime/imageruntime/pouch.go index 9d9b7ef5e4..f85957cd13 100644 --- a/pkg/daemon/criruntime/imageruntime/pouch.go +++ b/pkg/daemon/criruntime/imageruntime/pouch.go @@ -80,26 +80,26 @@ func (d *pouchImageService) PullImage(ctx context.Context, imageName, tag string registry := daemonutil.ParseRegistry(imageName) var ioReader io.ReadCloser - if len(pullSecrets) > 0 { - var authInfos []daemonutil.AuthInfo - authInfos, err = secret.ConvertToRegistryAuths(pullSecrets, registry) - if err == nil { - var pullErrs []error - for _, authInfo := range authInfos { - var pullErr error - klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username) - ioReader, pullErr = d.client.ImagePull(ctx, imageName, tag, authInfo.EncodeToString()) - if pullErr == nil { - return newImagePullStatusReader(ioReader), nil - } - d.handleRuntimeError(pullErr) - klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr) - pullErrs = append(pullErrs, pullErr) - } - if len(pullErrs) > 0 { - err = utilerrors.NewAggregate(pullErrs) + var authInfos []daemonutil.AuthInfo + authInfos, err = secret.ConvertToRegistryAuths(pullSecrets, registry) + if err == nil { + var pullErrs []error + for _, authInfo := range authInfos { + var pullErr error + klog.V(5).Infof("Pull image %v:%v with user %v", imageName, tag, authInfo.Username) + ioReader, pullErr = d.client.ImagePull(ctx, imageName, tag, authInfo.EncodeToString()) + if pullErr == nil { + return newImagePullStatusReader(ioReader), nil } + d.handleRuntimeError(pullErr) + klog.Warningf("Failed to pull image %v:%v with user %v, err %v", imageName, tag, authInfo.Username, pullErr) + pullErrs = append(pullErrs, pullErr) + } + if len(pullErrs) > 0 { + err = utilerrors.NewAggregate(pullErrs) } + } else { + klog.Errorf("Failed to convert to auth info for registry, err %v", err) } // Try the default secret diff --git a/pkg/util/secret/parse.go b/pkg/util/secret/parse.go index 5ed48b7309..d7334f1584 100644 --- a/pkg/util/secret/parse.go +++ b/pkg/util/secret/parse.go @@ -6,6 +6,7 @@ import ( daemonutil "github.com/openkruise/kruise/pkg/daemon/util" corev1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/credentialprovider" credentialprovidersecrets "k8s.io/kubernetes/pkg/credentialprovider/secrets" ) @@ -22,10 +23,19 @@ func AuthInfos(ctx context.Context, imageName, tag string, pullSecrets []corev1. } var ( - keyring = credentialprovider.NewDockerKeyring() + keyring credentialprovider.DockerKeyring ) +// make and set new docker keyring +func MakeAndSetKeyring() { + klog.Infof("make and set new docker keyring") + keyring = credentialprovider.NewDockerKeyring() +} + func ConvertToRegistryAuths(pullSecrets []corev1.Secret, repo string) (infos []daemonutil.AuthInfo, err error) { + if keyring == nil { + MakeAndSetKeyring() + } keyring, err := credentialprovidersecrets.MakeDockerKeyring(pullSecrets, keyring) if err != nil { return nil, err