Use ephemeral S3 buckets for E2E tests

tests/e2e/go.mod

@@ -5,6 +5,10 @@ go 1.23.5
 replace k8s.io/kops => ../../.
 
 require (
+	github.com/aws/aws-sdk-go-v2 v1.31.0
+	github.com/aws/aws-sdk-go-v2/config v1.27.38
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.63.2
+	github.com/aws/aws-sdk-go-v2/service/sts v1.31.2
 	github.com/blang/semver/v4 v4.0.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/octago/sflags v0.2.0
@@ -66,9 +70,7 @@ require (
 	github.com/aliyun/credentials-go v1.2.3 // indirect
 	github.com/apparentlymart/go-cidr v1.1.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.31.0 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.27.38 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.36 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 // indirect
@@ -82,10 +84,8 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 // indirect
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.63.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.23.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.31.2 // indirect
 	github.com/aws/smithy-go v1.21.0 // indirect
 	github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20220228164355-396b2034c795 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect

tests/e2e/kubetest2-kops/aws/s3.go

@@ -0,0 +1,194 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aws
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/aws-sdk-go-v2/service/sts"
+	"k8s.io/klog/v2"
+)
+
+// defaultRegion is the region to query the AWS APIs through, this can be any AWS region is required even if we are not
+// running on AWS.
+const defaultRegion = "us-east-2"
+
+// Client contains S3 and STS clients that are used to perform bucket and object actions.
+type Client struct {
+	s3Client  *s3.Client
+	stsClient *sts.Client
+}
+
+// NewAWSClient returns a new instance of awsClient configured to work in the default region (us-east-2).
+func NewClient(ctx context.Context) (*Client, error) {
+	cfg, err := awsconfig.LoadDefaultConfig(ctx,
+		awsconfig.WithRegion(defaultRegion))
+	if err != nil {
+		return nil, fmt.Errorf("loading AWS config: %w", err)
+	}
+
+	return &Client{
+		s3Client:  s3.NewFromConfig(cfg),
+		stsClient: sts.NewFromConfig(cfg),
+	}, nil
+}
+
+// BucketName constructs an unique bucket name using the AWS account ID in the default region (us-east-2).
+func (c Client) BucketName(ctx context.Context) (string, error) {
+	// Construct the bucket name based on the ProwJob ID (if running in Prow) or AWS account ID (if running outside
+	// Prow) and the current timestamp
+	var identifier string
+	if jobID := os.Getenv("PROW_JOB_ID"); len(jobID) >= 4 {
+		identifier = jobID[:4]
+	} else {
+		callerIdentity, err := c.stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{})
+		if err != nil {
+			return "", fmt.Errorf("building AWS STS presigned request: %w", err)
+		}
+		identifier = *callerIdentity.Account
+	}
+	timestamp := time.Now().Format("20060102150405")
+	bucket := fmt.Sprintf("k8s-infra-kops-%s-%s", identifier, timestamp)
+
+	bucket = strings.ToLower(bucket)
+	// Only allow lowercase letters, numbers, and hyphens
+	bucket = regexp.MustCompile("[^a-z0-9-]").ReplaceAllString(bucket, "")
+
+	if len(bucket) > 63 {
+		bucket = bucket[:63] // Max length is 63
+	}
+
+	return bucket, nil
+}
+
+// EnsureS3Bucket creates a new S3 bucket with the given name and public read permissions.
+func (c Client) EnsureS3Bucket(ctx context.Context, bucketName string, publicRead bool) error {
+	bucketName = strings.TrimPrefix(bucketName, "s3://")
+	_, err := c.s3Client.CreateBucket(ctx, &s3.CreateBucketInput{
+		Bucket: aws.String(bucketName),
+		CreateBucketConfiguration: &types.CreateBucketConfiguration{
+			LocationConstraint: defaultRegion,
+		},
+	})
+	if err != nil {
+		var exists *types.BucketAlreadyExists
+		if errors.As(err, &exists) {
+			klog.Infof("Bucket %s already exists\n", bucketName)
+		} else {
+			klog.Infof("Error creating bucket %s, err: %v\n", bucketName, err)
+		}
+
+		return fmt.Errorf("creating bucket %s: %w", bucketName, err)
+	}
+
+	// Wait for the bucket to be created
+	err = s3.NewBucketExistsWaiter(c.s3Client).Wait(
+		ctx, &s3.HeadBucketInput{
+			Bucket: aws.String(bucketName),
+		},
+		time.Minute)
+	if err != nil {
+		klog.Infof("Failed attempt to wait for bucket %s to exist, err: %v", bucketName, err)
+
+		return fmt.Errorf("waiting for bucket %s to exist: %w", bucketName, err)
+	}
+
+	klog.Infof("Bucket %s created successfully", bucketName)
+
+	if publicRead {
+		err = c.setPublicReadPolicy(ctx, bucketName)
+		if err != nil {
+			klog.Errorf("Failed to set public read policy on bucket %s, err: %v", bucketName, err)
+
+			return fmt.Errorf("setting public read policy for bucket %s: %w", bucketName, err)
+		}
+
+		klog.Infof("Public read policy set on bucket %s", bucketName)
+	}
+
+	return nil
+}
+
+// DeleteS3Bucket deletes a S3 bucket with the given name.
+func (c Client) DeleteS3Bucket(ctx context.Context, bucketName string) error {
+	bucketName = strings.TrimPrefix(bucketName, "s3://")
+	_, err := c.s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{
+		Bucket: aws.String(bucketName),
+	})
+	if err != nil {
+		var noBucket *types.NoSuchBucket
+		if errors.As(err, &noBucket) {
+			klog.Infof("Bucket %s does not exits.", bucketName)
+
+			return nil
+		} else {
+			klog.Infof("Couldn't delete bucket %s, err: %v", bucketName, err)
+
+			return fmt.Errorf("deleting bucket %s: %w", bucketName, err)
+		}
+	}
+
+	err = s3.NewBucketNotExistsWaiter(c.s3Client).Wait(
+		ctx, &s3.HeadBucketInput{
+			Bucket: aws.String(bucketName),
+		},
+		time.Minute)
+	if err != nil {
+		klog.Infof("Failed attempt to wait for bucket %s to be deleted, err: %v", bucketName, err)
+
+		return fmt.Errorf("waiting for bucket %s to be deleted, err: %w", bucketName, err)
+	}
+
+	klog.Infof("Bucket %s deleted", bucketName)
+
+	return nil
+}
+
+func (c Client) setPublicReadPolicy(ctx context.Context, bucketName string) error {
+	policy := fmt.Sprintf(`{
+    "Version": "2012-10-17",
+    "Statement": [
+      {
+        "Sid": "PublicReadGetObject",
+        "Effect": "Allow",
+        "Principal": "*",
+        "Action": "s3:GetObject",
+        "Resource": "arn:aws:s3:::%s/*"
+      }
+    ]
+  }`, bucketName)
+
+	_, err := c.s3Client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
+		Bucket: aws.String(bucketName),
+		Policy: aws.String(policy),
+	})
+	if err != nil {
+		return fmt.Errorf("putting bucket policy for %s: %w", bucketName, err)
+	}
+
+	return nil
+}

tests/e2e/kubetest2-kops/deployer/common.go

@@ -17,13 +17,15 @@ limitations under the License.
 package deployer
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"k8s.io/klog/v2"
+	"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
 	"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
 	"k8s.io/kops/tests/e2e/pkg/target"
 	"k8s.io/kops/tests/e2e/pkg/util"
@@ -51,6 +53,12 @@ func (d *deployer) initialize() error {
 
 	switch d.CloudProvider {
 	case "aws":
+		client, err := aws.NewClient(context.Background())
+		if err != nil {
+			return fmt.Errorf("init failed to build AWS client: %w", err)
+		}
+		d.aws = client
+
 		if d.SSHPrivateKeyPath == "" {
 			d.SSHPrivateKeyPath = os.Getenv("AWS_SSH_PRIVATE_KEY_FILE")
 		}
@@ -316,34 +324,53 @@ func defaultClusterName(cloudProvider string) (string, error) {
 // stateStore returns the kops state store to use
 // defaulting to values used in prow jobs
 func (d *deployer) stateStore() string {
+	if d.stateStoreName != "" {
+		return d.stateStoreName
+	}
 	ss := os.Getenv("KOPS_STATE_STORE")
 	if ss == "" {
 		switch d.CloudProvider {
 		case "aws":
-			ss = "s3://k8s-kops-prow"
+			ctx := context.Background()
+			bucketName, err := d.aws.BucketName(ctx)
+			if err != nil {
+				klog.Fatalf("Failed to generate bucket name: %v", err)
+				return ""
+			}
+			d.createBucket = true
+			ss = "s3://" + bucketName
 		case "gce":
 			d.createBucket = true
 			ss = "gs://" + gce.GCSBucketName(d.GCPProject, "state")
 		case "digitalocean":
 			ss = "do://e2e-kops-space"
 		}
 	}
+
+	d.stateStoreName = ss
 	return ss
 }
 
 // discoveryStore returns the VFS path to use for public OIDC documents
 func (d *deployer) discoveryStore() string {
+	if d.discoveryStoreName != "" {
+		return d.discoveryStoreName
+	}
 	discovery := os.Getenv("KOPS_DISCOVERY_STORE")
 	if discovery == "" {
 		switch d.CloudProvider {
 		case "aws":
 			discovery = "s3://k8s-kops-ci-prow"
 		}
 	}
+	d.discoveryStoreName = discovery
 	return discovery
 }
 
 func (d *deployer) stagingStore() string {
+	if d.stagingStoreName != "" {
+		return d.stagingStoreName
+	}
 	sb := os.Getenv("KOPS_STAGING_BUCKET")
 	if sb == "" {
 		switch d.CloudProvider {
@@ -352,6 +379,7 @@ func (d *deployer) stagingStore() string {
 			sb = "gs://" + gce.GCSBucketName(d.GCPProject, "staging")
 		}
 	}
+	d.stagingStoreName = sb
 	return sb
 }
 

tests/e2e/kubetest2-kops/deployer/deployer.go

@@ -25,6 +25,7 @@ import (
 	"github.com/octago/sflags/gen/gpflag"
 	"github.com/spf13/pflag"
 	"k8s.io/klog/v2"
+	"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
 	"k8s.io/kops/tests/e2e/kubetest2-kops/builder"
 	"k8s.io/kops/tests/e2e/pkg/target"
 
@@ -57,7 +58,6 @@ type deployer struct {
 	CreateArgs             string   `flag:"create-args" desc:"Extra space-separated arguments passed to 'kops create cluster'"`
 	KopsBinaryPath         string   `flag:"kops-binary-path" desc:"The path to kops executable used for testing"`
 	KubernetesFeatureGates string   `flag:"kubernetes-feature-gates" desc:"Feature Gates to enable on Kubernetes components"`
-	createBucket           bool     `flag:"-"`
 
 	// ControlPlaneCount specifies the number of VMs in the control-plane.
 	ControlPlaneCount int `flag:"control-plane-count" desc:"Number of control-plane instances"`
@@ -90,6 +90,13 @@ type deployer struct {
 	manifestPath string
 	terraform    *target.Terraform
 
+	aws *aws.Client
+
+	createBucket       bool
+	stateStoreName     string
+	discoveryStoreName string
+	stagingStoreName   string
+
 	// boskos struct field will be non-nil when the deployer is
 	// using boskos to acquire a GCP project
 	boskos                  *client.Client
@@ -106,8 +113,10 @@ type deployer struct {
 var _ types.NewDeployer = New
 
 // assert that deployer implements types.Deployer
-var _ types.Deployer = &deployer{}
-var _ types.DeployerWithPostTester = &deployer{}
+var (
+	_ types.Deployer               = &deployer{}
+	_ types.DeployerWithPostTester = &deployer{}
+)
 
 func (d *deployer) Provider() string {
 	return Name

tests/e2e/kubetest2-kops/deployer/down.go

@@ -17,6 +17,7 @@ limitations under the License.
 package deployer
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
@@ -72,9 +73,17 @@ func (d *deployer) Down() error {
 		return err
 	}
 
-	if d.CloudProvider == "gce" && d.createBucket {
-		gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
-		gce.DeleteGCSBucket(d.stagingStore(), d.GCPProject)
+	if d.createBucket {
+		switch d.CloudProvider {
+		case "aws":
+			ctx := context.Background()
+			if err := d.aws.DeleteS3Bucket(ctx, d.stateStore()); err != nil {
+				return err
+			}
+		case "gce":
+			gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
+			gce.DeleteGCSBucket(d.stagingStore(), d.GCPProject)
+		}
 	}
 
 	if d.boskos != nil {