create a dedicated staging bucket for kops builds

kubernetes · Oct 6, 2023 · f1c3d66 · f1c3d66
1 parent d630f9c
commit f1c3d66
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 13 deletions.
diff --git a/tests/e2e/kubetest2-kops/deployer/build.go b/tests/e2e/kubetest2-kops/deployer/build.go
@@ -24,13 +24,14 @@ import (
 	"strings"
 
 	"k8s.io/klog/v2"
+	"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
 	"k8s.io/kops/tests/e2e/pkg/util"
 	"sigs.k8s.io/kubetest2/pkg/exec"
 )
 
 const (
 	defaultJobName = "pull-kops-e2e-kubernetes-aws"
-	defaultGCSPath = "gs://kops-ci/pulls/%v/pull-%v"
+	defaultGCSPath = "gs://k8s-staging-kops/pulls/%v/pull-%v"
 )
 
 func (d *deployer) Build() error {
@@ -66,6 +67,12 @@ func (d *deployer) verifyBuildFlags() error {
 		if !strings.HasPrefix(d.StageLocation, "gs://") {
 			return errors.New("stage-location must be a gs:// path")
 		}
+	} else if d.boskos != nil {
+		d.StageLocation = d.stagingStore()
+		klog.Infof("creating staging bucket %s to hold kops build artifacts", d.StageLocation)
+		if err := gce.EnsureGCSBucket(d.StageLocation, d.GCPProject, true); err != nil {
+			return err
+		}
 	} else {
 		stageLocation, err := defaultStageLocation(d.KopsRoot)
 		if err != nil {

diff --git a/tests/e2e/kubetest2-kops/deployer/common.go b/tests/e2e/kubetest2-kops/deployer/common.go
@@ -41,11 +41,6 @@ func (d *deployer) init() error {
 
 // initialize should only be called by init(), behind a sync.Once
 func (d *deployer) initialize() error {
-	if d.commonOptions.ShouldBuild() {
-		if err := d.verifyBuildFlags(); err != nil {
-			return fmt.Errorf("init failed to check build flags: %v", err)
-		}
-	}
 	if d.commonOptions.ShouldUp() || d.commonOptions.ShouldDown() {
 		if err := d.verifyKopsFlags(); err != nil {
 			return fmt.Errorf("init failed to check kops flags: %v", err)
@@ -118,6 +113,12 @@ func (d *deployer) initialize() error {
 		}
 	}
 
+	if d.commonOptions.ShouldBuild() {
+		if err := d.verifyBuildFlags(); err != nil {
+			return fmt.Errorf("init failed to check build flags: %v", err)
+		}
+	}
+
 	if d.SSHUser == "" {
 		d.SSHUser = os.Getenv("KUBE_SSH_USER")
 	}
@@ -272,14 +273,26 @@ func (d *deployer) stateStore() string {
 			ss = "s3://k8s-kops-prow"
 		case "gce":
 			d.createBucket = true
-			ss = "gs://" + gce.GCSBucketName(d.GCPProject)
+			ss = "gs://" + gce.GCSBucketName(d.GCPProject, "state")
 		case "digitalocean":
 			ss = "do://e2e-kops-space"
 		}
 	}
 	return ss
 }
 
+func (d *deployer) stagingStore() string {
+	sb := os.Getenv("KOPS_STAGING_BUCKET")
+	if sb == "" {
+		switch d.CloudProvider {
+		case "gce":
+			d.createBucket = true
+			sb = "gs://" + gce.GCSBucketName(d.GCPProject, "staging")
+		}
+	}
+	return sb
+}
+
 // the default is $ARTIFACTS if set, otherwise ./_artifacts
 // constructed as an absolute path to help the ginkgo tester because
 // for some reason it needs an absolute path to the kubeconfig

diff --git a/tests/e2e/kubetest2-kops/deployer/down.go b/tests/e2e/kubetest2-kops/deployer/down.go
@@ -56,6 +56,7 @@ func (d *deployer) Down() error {
 
 	if d.CloudProvider == "gce" && d.createBucket {
 		gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
+		gce.DeleteGCSBucket(d.stagingStore(), d.GCPProject)
 	}
 
 	if d.boskos != nil {

diff --git a/tests/e2e/kubetest2-kops/deployer/up.go b/tests/e2e/kubetest2-kops/deployer/up.go
@@ -49,7 +49,7 @@ func (d *deployer) Up() error {
 	}
 
 	if d.CloudProvider == "gce" && d.createBucket {
-		if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject); err != nil {
+		if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject, false); err != nil {
 			return err
 		}
 	}

diff --git a/tests/e2e/kubetest2-kops/gce/gcs.go b/tests/e2e/kubetest2-kops/gce/gcs.go
@@ -21,25 +21,26 @@ import (
 	"encoding/hex"
 	"os"
 	"strings"
+	"time"
 
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/kubetest2/pkg/exec"
 )
 
-func GCSBucketName(projectID string) string {
+func GCSBucketName(projectID, prefix string) string {
 	var s string
-	if jobID := os.Getenv("PROW_JOB_ID"); len(jobID) >= 2 {
-		s = jobID[:2]
+	if jobID := os.Getenv("PROW_JOB_ID"); len(jobID) >= 4 {
+		s = jobID[:4]
 	} else {
 		b := make([]byte, 2)
 		rand.Read(b)
 		s = hex.EncodeToString(b)
 	}
-	bucket := strings.Join([]string{projectID, "state", s}, "-")
+	bucket := strings.Join([]string{projectID, prefix, s}, "-")
 	return bucket
 }
 
-func EnsureGCSBucket(bucketPath, projectID string) error {
+func EnsureGCSBucket(bucketPath, projectID string, public bool) error {
 	lsArgs := []string{
 		"gsutil", "ls", "-b",
 	}
@@ -75,6 +76,22 @@ func EnsureGCSBucket(bucketPath, projectID string) error {
 	if err != nil {
 		return err
 	}
+
+	if public {
+		iamArgs := []string{
+			"gsutil", "iam", "ch", "allUsers:objectViewer",
+		}
+		iamArgs = append(iamArgs, bucketPath)
+		klog.Info(strings.Join(iamArgs, " "))
+		// GCS APIs are strongly consistent but this should help with flakes
+		time.Sleep(10 * time.Second)
+		cmd = exec.Command(iamArgs[0], iamArgs[1:]...)
+		exec.InheritOutput(cmd)
+		err = cmd.Run()
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }