From f1c3d6659d43359610c0c47d78f9b97d2c487538 Mon Sep 17 00:00:00 2001 From: upodroid Date: Fri, 6 Oct 2023 17:51:17 +0100 Subject: [PATCH] create a dedicated staging bucket for kops builds --- tests/e2e/kubetest2-kops/deployer/build.go | 9 ++++++- tests/e2e/kubetest2-kops/deployer/common.go | 25 ++++++++++++++----- tests/e2e/kubetest2-kops/deployer/down.go | 1 + tests/e2e/kubetest2-kops/deployer/up.go | 2 +- tests/e2e/kubetest2-kops/gce/gcs.go | 27 +++++++++++++++++---- 5 files changed, 51 insertions(+), 13 deletions(-) diff --git a/tests/e2e/kubetest2-kops/deployer/build.go b/tests/e2e/kubetest2-kops/deployer/build.go index 421742c01124a..49602b5ab02f1 100644 --- a/tests/e2e/kubetest2-kops/deployer/build.go +++ b/tests/e2e/kubetest2-kops/deployer/build.go @@ -24,13 +24,14 @@ import ( "strings" "k8s.io/klog/v2" + "k8s.io/kops/tests/e2e/kubetest2-kops/gce" "k8s.io/kops/tests/e2e/pkg/util" "sigs.k8s.io/kubetest2/pkg/exec" ) const ( defaultJobName = "pull-kops-e2e-kubernetes-aws" - defaultGCSPath = "gs://kops-ci/pulls/%v/pull-%v" + defaultGCSPath = "gs://k8s-staging-kops/pulls/%v/pull-%v" ) func (d *deployer) Build() error { @@ -66,6 +67,12 @@ func (d *deployer) verifyBuildFlags() error { if !strings.HasPrefix(d.StageLocation, "gs://") { return errors.New("stage-location must be a gs:// path") } + } else if d.boskos != nil { + d.StageLocation = d.stagingStore() + klog.Infof("creating staging bucket %s to hold kops build artifacts", d.StageLocation) + if err := gce.EnsureGCSBucket(d.StageLocation, d.GCPProject, true); err != nil { + return err + } } else { stageLocation, err := defaultStageLocation(d.KopsRoot) if err != nil { diff --git a/tests/e2e/kubetest2-kops/deployer/common.go b/tests/e2e/kubetest2-kops/deployer/common.go index 9dc7341f5dc12..df52902ec7294 100644 --- a/tests/e2e/kubetest2-kops/deployer/common.go +++ b/tests/e2e/kubetest2-kops/deployer/common.go @@ -41,11 +41,6 @@ func (d *deployer) init() error { // initialize should only be called by init(), behind a sync.Once func (d *deployer) initialize() error { - if d.commonOptions.ShouldBuild() { - if err := d.verifyBuildFlags(); err != nil { - return fmt.Errorf("init failed to check build flags: %v", err) - } - } if d.commonOptions.ShouldUp() || d.commonOptions.ShouldDown() { if err := d.verifyKopsFlags(); err != nil { return fmt.Errorf("init failed to check kops flags: %v", err) @@ -118,6 +113,12 @@ func (d *deployer) initialize() error { } } + if d.commonOptions.ShouldBuild() { + if err := d.verifyBuildFlags(); err != nil { + return fmt.Errorf("init failed to check build flags: %v", err) + } + } + if d.SSHUser == "" { d.SSHUser = os.Getenv("KUBE_SSH_USER") } @@ -272,7 +273,7 @@ func (d *deployer) stateStore() string { ss = "s3://k8s-kops-prow" case "gce": d.createBucket = true - ss = "gs://" + gce.GCSBucketName(d.GCPProject) + ss = "gs://" + gce.GCSBucketName(d.GCPProject, "state") case "digitalocean": ss = "do://e2e-kops-space" } @@ -280,6 +281,18 @@ func (d *deployer) stateStore() string { return ss } +func (d *deployer) stagingStore() string { + sb := os.Getenv("KOPS_STAGING_BUCKET") + if sb == "" { + switch d.CloudProvider { + case "gce": + d.createBucket = true + sb = "gs://" + gce.GCSBucketName(d.GCPProject, "staging") + } + } + return sb +} + // the default is $ARTIFACTS if set, otherwise ./_artifacts // constructed as an absolute path to help the ginkgo tester because // for some reason it needs an absolute path to the kubeconfig diff --git a/tests/e2e/kubetest2-kops/deployer/down.go b/tests/e2e/kubetest2-kops/deployer/down.go index d35d05d21c4aa..8b0a52e6ff4a1 100644 --- a/tests/e2e/kubetest2-kops/deployer/down.go +++ b/tests/e2e/kubetest2-kops/deployer/down.go @@ -56,6 +56,7 @@ func (d *deployer) Down() error { if d.CloudProvider == "gce" && d.createBucket { gce.DeleteGCSBucket(d.stateStore(), d.GCPProject) + gce.DeleteGCSBucket(d.stagingStore(), d.GCPProject) } if d.boskos != nil { diff --git a/tests/e2e/kubetest2-kops/deployer/up.go b/tests/e2e/kubetest2-kops/deployer/up.go index 872fbf6127dcd..cfbdfb941007c 100644 --- a/tests/e2e/kubetest2-kops/deployer/up.go +++ b/tests/e2e/kubetest2-kops/deployer/up.go @@ -49,7 +49,7 @@ func (d *deployer) Up() error { } if d.CloudProvider == "gce" && d.createBucket { - if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject); err != nil { + if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject, false); err != nil { return err } } diff --git a/tests/e2e/kubetest2-kops/gce/gcs.go b/tests/e2e/kubetest2-kops/gce/gcs.go index 45cf93fd2a713..067290d72057c 100644 --- a/tests/e2e/kubetest2-kops/gce/gcs.go +++ b/tests/e2e/kubetest2-kops/gce/gcs.go @@ -21,25 +21,26 @@ import ( "encoding/hex" "os" "strings" + "time" "k8s.io/klog/v2" "sigs.k8s.io/kubetest2/pkg/exec" ) -func GCSBucketName(projectID string) string { +func GCSBucketName(projectID, prefix string) string { var s string - if jobID := os.Getenv("PROW_JOB_ID"); len(jobID) >= 2 { - s = jobID[:2] + if jobID := os.Getenv("PROW_JOB_ID"); len(jobID) >= 4 { + s = jobID[:4] } else { b := make([]byte, 2) rand.Read(b) s = hex.EncodeToString(b) } - bucket := strings.Join([]string{projectID, "state", s}, "-") + bucket := strings.Join([]string{projectID, prefix, s}, "-") return bucket } -func EnsureGCSBucket(bucketPath, projectID string) error { +func EnsureGCSBucket(bucketPath, projectID string, public bool) error { lsArgs := []string{ "gsutil", "ls", "-b", } @@ -75,6 +76,22 @@ func EnsureGCSBucket(bucketPath, projectID string) error { if err != nil { return err } + + if public { + iamArgs := []string{ + "gsutil", "iam", "ch", "allUsers:objectViewer", + } + iamArgs = append(iamArgs, bucketPath) + klog.Info(strings.Join(iamArgs, " ")) + // GCS APIs are strongly consistent but this should help with flakes + time.Sleep(10 * time.Second) + cmd = exec.Command(iamArgs[0], iamArgs[1:]...) + exec.InheritOutput(cmd) + err = cmd.Run() + if err != nil { + return err + } + } return nil }