Keep an up-to-date copy of the KMS master key

Validate KMS master key using a test of encrypting decryption, in order to startup MinIO

cmd/common-main.go

@@ -962,12 +962,23 @@ func handleKMSConfig() {
 		if err != nil {
 			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
 		}
-		// We check that the default key ID exists or try to create it otherwise.
-		// This implicitly checks that we can communicate to KES. We don't treat
-		// a policy error as failure condition since MinIO may not have the permission
+		// Try to generate/decrypt a data encryption key. Only try to create key if this fails.
+		// This implicitly checks that we can communicate to KES.
+		// We don't treat a policy error as failure condition since MinIO may not have the permission
 		// to create keys - just to generate/decrypt data encryption keys.
-		if err = KMS.CreateKey(context.Background(), env.Get(kms.EnvKESKeyName, "")); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
-			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
+		ctx := context.WithValue(context.Background(), "ValidateKeyOnly", "true")
+		results := KMS.Verify(ctx)
+		validKey := false
+		for _, result := range results {
+			if validKey == false && result.Status == "online" && result.Decrypt == "success" && result.Encrypt == "success" {
+				validKey = true
+				break
+			}
+		}
+		if !validKey {
+			if err = KMS.CreateKey(context.Background(), env.Get(kms.EnvKESKeyName, "")); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
+				logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
+			}
 		}
 		GlobalKMS = KMS
 	}

internal/kms/kes.go

@@ -25,12 +25,12 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
-	"strings"
-	"sync"
-
 	"github.com/minio/kms-go/kes"
 	"github.com/minio/pkg/v2/certs"
 	"github.com/minio/pkg/v2/env"
+	"strings"
+	"sync"
+	"time"
 )
 
 const (
@@ -138,9 +138,33 @@ func NewWithConfig(config Config) (KMS, error) {
 			}
 		}
 	}()
+
+	go func() {
+		c.keepKeyInCache()
+	}()
 	return c, nil
 }
 
+// Request KES keep an up-to-date copy of the KMS master key to allow minio to start up even if KMS is down. The
+// cached key may still be evicted if the period of this function is longer than that of KES .cache.expiry.unused
+func (c *kesClient) keepKeyInCache() {
+	ctx := context.WithValue(context.Background(), "ValidateKeyOnly", "true")
+
+	ticker := time.NewTicker(10 * time.Second)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ticker.C:
+		case <-ctx.Done():
+			return
+		}
+		results := c.Verify(ctx)
+		for _, result := range results {
+			fmt.Println(result.Endpoint, result.Status, result.Version, result.Decrypt, result.Encrypt)
+		}
+	}
+}
+
 type kesClient struct {
 	lock         sync.RWMutex
 	defaultKeyID string
@@ -416,17 +440,23 @@ func (c *kesClient) Verify(ctx context.Context) []VerifyResult {
 			HTTPClient: c.client.HTTPClient,
 		}
 
-		// 1. Get stats for the KES instance
-		state, err := client.Status(ctx)
-		if err != nil {
-			results = append(results, VerifyResult{Status: "offline", Endpoint: endpoint})
-			continue
+		state := kes.State{}
+		// Permission on /status is not required to validate kes key encryption and decryption
+		if ctx.Value("ValidateKeyOnly") != "true" {
+			// 1. Get stats for the KES instance
+			clientState, err := client.Status(ctx)
+			if err != nil {
+				results = append(results, VerifyResult{Status: "offline", Endpoint: endpoint})
+				continue
+			}
+			state = clientState
 		}
 
 		// 2. Generate a new key using the KMS.
 		kmsCtx, err := kmsContext.MarshalText()
 		if err != nil {
 			results = append(results, VerifyResult{Status: "offline", Endpoint: endpoint})
+			fmt.Println(err)
 			continue
 		}
 		result := VerifyResult{Status: "online", Endpoint: endpoint, Version: state.Version}