Skip to content

refactor(gsutil): Migrate gsutil to gcloud storage #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
googlyrahman wants to merge 1 commit into from
Closed

refactor(gsutil): Migrate gsutil to gcloud storage #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions .ci/magician/cmd/vcr_cassette_update.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,8 @@ func execVCRCassetteUpdate(buildID, today string, rnr ExecRunner, ctlr *source.C
// main cassettes backup
// incase nightly run goes wrong. this will be used to restore the cassettes
cassettePath := vt.CassettePath(provider.Beta)
args := []string{"-m", "-q", "cp", filepath.Join(cassettePath, "*"), bucketPrefix + "/main_cassettes_backup/fixtures/"}
if _, err := rnr.Run("gsutil", args, nil); err != nil {
args := []string{"storage", "cp", filepath.Join(cassettePath, "*"), bucketPrefix + "/main_cassettes_backup/fixtures/"}
if _, err := rnr.Run("gcloud", args, nil); err != nil {
return fmt.Errorf("error backup cassettes: %w", err)
}

Expand Down Expand Up @@ -222,18 +222,19 @@ func execVCRCassetteUpdate(buildID, today string, rnr ExecRunner, ctlr *source.C
}

func uploadLogsToGCS(src, dest string, rnr ExecRunner) (string, error) {
return uploadToGCS(src, dest, []string{"-h", "Content-Type:text/plain", "-q", "cp", "-r"}, rnr)
// TODO: Flag "-h" is not supported in "gcloud storage cp". Manual review required.
return uploadToGCS(src, dest, []string{"storage", "cp", "--recursive"}, rnr)
}

func uploadCassettesToGCS(src, dest string, rnr ExecRunner) (string, error) {
return uploadToGCS(src, dest, []string{"-m", "-q", "cp"}, rnr)
return uploadToGCS(src, dest, []string{"storage", "cp"}, rnr)
}

func uploadToGCS(src, dest string, opts []string, rnr ExecRunner) (string, error) {
fmt.Printf("uploading from %s to %s\n", src, dest)
args := append(opts, src, dest)
fmt.Println("gsutil", args)
return rnr.Run("gsutil", args, nil)
fmt.Println("gcloud", args)
return rnr.Run("gcloud", args, nil)
}

func formatVCRCassettesUpdateReplaying(data vcrCassetteUpdateReplayingResult) (string, error) {
Expand Down
22 changes: 11 additions & 11 deletions .ci/magician/cmd/vcr_cassette_update_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -313,8 +313,8 @@ func TestExecVCRCassetteUpdate(t *testing.T) {
cmdResults: make(map[string]string),
expectedCalls: map[string][]ParameterList{
"Run": {
{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-m", "-q", "cp", "gs://ci-vcr-cassettes/beta/fixtures/*", "/mock/dir/magic-modules/.ci/magician/cassettes/beta"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-m", "-q", "cp", "/mock/dir/magic-modules/.ci/magician/cassettes/beta/*", "gs://vcr-nightly/beta/2024-07-08/buildID/main_cassettes_backup/fixtures/"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gcloud", []string{"storage", "cp", "gs://ci-vcr-cassettes/beta/fixtures/*", "/mock/dir/magic-modules/.ci/magician/cassettes/beta"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gcloud", []string{"storage", "cp", "/mock/dir/magic-modules/.ci/magician/cassettes/beta/*", "gs://vcr-nightly/beta/2024-07-08/buildID/main_cassettes_backup/fixtures/"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "git", []string{"clone", "https://hashicorp:token@github.com/hashicorp/terraform-provider-google-beta", "gopath/src/github.com/hashicorp/terraform-provider-google-beta"}, map[string]string(nil)},
{"gopath/src/github.com/hashicorp/terraform-provider-google-beta", "go", []string{"list", "./..."}, map[string]string(nil)},
{"gopath/src/github.com/hashicorp/terraform-provider-google-beta", "go", []string{"test", "", "-parallel", "32", "-v", "-run=TestAcc", "-timeout", "240m", "-ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc", "-vet=off"}, map[string]string{
Expand All @@ -332,8 +332,8 @@ func TestExecVCRCassetteUpdate(t *testing.T) {
"VCR_MODE": "REPLAYING",
"VCR_PATH": "/mock/dir/magic-modules/.ci/magician/cassettes/beta",
}},
{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-h", "Content-Type:text/plain", "-q", "cp", "-r", "/mock/dir/magic-modules/.ci/magician/testlogs/replaying_test.log", "gs://vcr-nightly/beta/2024-07-08/buildID/logs/replaying/"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-h", "Content-Type:text/plain", "-q", "cp", "-r", "/mock/dir/magic-modules/.ci/magician/testlogs/replaying/beta/*", "gs://vcr-nightly/beta/2024-07-08/buildID/logs/build-log/"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gcloud", []string{"storage", "cp", "--content-type=text/plain", "--recursive", "/mock/dir/magic-modules/.ci/magician/testlogs/replaying_test.log", "gs://vcr-nightly/beta/2024-07-08/buildID/logs/replaying/"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gcloud", []string{"storage", "cp", "--content-type=text/plain", "--recursive", "/mock/dir/magic-modules/.ci/magician/testlogs/replaying/beta/*", "gs://vcr-nightly/beta/2024-07-08/buildID/logs/build-log/"}, map[string]string(nil)},
},
},
},
Expand All @@ -346,8 +346,8 @@ func TestExecVCRCassetteUpdate(t *testing.T) {
expectedCalls: map[string][]ParameterList{
"Run": {
// replay
{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-m", "-q", "cp", "gs://ci-vcr-cassettes/beta/fixtures/*", "/mock/dir/magic-modules/.ci/magician/cassettes/beta"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-m", "-q", "cp", "/mock/dir/magic-modules/.ci/magician/cassettes/beta/*", "gs://vcr-nightly/beta/2024-07-08/buildID/main_cassettes_backup/fixtures/"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gcloud", []string{"storage", "cp", "gs://ci-vcr-cassettes/beta/fixtures/*", "/mock/dir/magic-modules/.ci/magician/cassettes/beta"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gcloud", []string{"storage", "cp", "/mock/dir/magic-modules/.ci/magician/cassettes/beta/*", "gs://vcr-nightly/beta/2024-07-08/buildID/main_cassettes_backup/fixtures/"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "git", []string{"clone", "https://hashicorp:token@github.com/hashicorp/terraform-provider-google-beta", "gopath/src/github.com/hashicorp/terraform-provider-google-beta"}, map[string]string(nil)},
{"gopath/src/github.com/hashicorp/terraform-provider-google-beta", "go", []string{"list", "./..."}, map[string]string(nil)},
{"gopath/src/github.com/hashicorp/terraform-provider-google-beta", "go", []string{"test", "", "-parallel", "32", "-v", "-run=TestAcc", "-timeout", "240m", "-ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc", "-vet=off"}, map[string]string{
Expand All @@ -365,8 +365,8 @@ func TestExecVCRCassetteUpdate(t *testing.T) {
"VCR_MODE": "REPLAYING",
"VCR_PATH": "/mock/dir/magic-modules/.ci/magician/cassettes/beta",
}},
{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-h", "Content-Type:text/plain", "-q", "cp", "-r", "/mock/dir/magic-modules/.ci/magician/testlogs/replaying_test.log", "gs://vcr-nightly/beta/2024-07-08/buildID/logs/replaying/"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-h", "Content-Type:text/plain", "-q", "cp", "-r", "/mock/dir/magic-modules/.ci/magician/testlogs/replaying/beta/*", "gs://vcr-nightly/beta/2024-07-08/buildID/logs/build-log/"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gcloud", []string{"storage", "cp", "--content-type=text/plain", "--recursive", "/mock/dir/magic-modules/.ci/magician/testlogs/replaying_test.log", "gs://vcr-nightly/beta/2024-07-08/buildID/logs/replaying/"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gcloud", []string{"storage", "cp", "--content-type=text/plain", "--recursive", "/mock/dir/magic-modules/.ci/magician/testlogs/replaying/beta/*", "gs://vcr-nightly/beta/2024-07-08/buildID/logs/build-log/"}, map[string]string(nil)},
// record
{"gopath/src/github.com/hashicorp/terraform-provider-google-beta", "go", []string{"list", "./..."}, map[string]string(nil)},
{"gopath/src/github.com/hashicorp/terraform-provider-google-beta", "go", []string{"test", "", "-parallel", "1", "-v", "-run=TestAccContainerNodePool_defaultDriverInstallation$", "-timeout", "240m", "-ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc", "-vet=off"}, map[string]string{
Expand All @@ -384,9 +384,9 @@ func TestExecVCRCassetteUpdate(t *testing.T) {
"VCR_MODE": "RECORDING",
"VCR_PATH": "/mock/dir/magic-modules/.ci/magician/cassettes/beta",
}},
{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-h", "Content-Type:text/plain", "-q", "cp", "-r", "/mock/dir/magic-modules/.ci/magician/testlogs/recording_test.log", "gs://vcr-nightly/beta/2024-07-08/buildID/logs/recording/"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-h", "Content-Type:text/plain", "-q", "cp", "-r", "/mock/dir/magic-modules/.ci/magician/testlogs/recording/beta/*", "gs://vcr-nightly/beta/2024-07-08/buildID/logs/build-log/"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-m", "-q", "cp", "/mock/dir/magic-modules/.ci/magician/cassettes/beta/*", "gs://ci-vcr-cassettes/beta/fixtures/"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gcloud", []string{"storage", "cp", "--content-type=text/plain", "--recursive", "/mock/dir/magic-modules/.ci/magician/testlogs/recording_test.log", "gs://vcr-nightly/beta/2024-07-08/buildID/logs/recording/"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gcloud", []string{"storage", "cp", "--content-type=text/plain", "--recursive", "/mock/dir/magic-modules/.ci/magician/testlogs/recording/beta/*", "gs://vcr-nightly/beta/2024-07-08/buildID/logs/build-log/"}, map[string]string(nil)},
{"/mock/dir/magic-modules/.ci/magician", "gcloud", []string{"storage", "cp", "/mock/dir/magic-modules/.ci/magician/cassettes/beta/*", "gs://ci-vcr-cassettes/beta/fixtures/"}, map[string]string(nil)},
},
},
},
Expand Down
21 changes: 11 additions & 10 deletions .ci/magician/cmd/vcr_merge.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ var vcrMergeCmd = &cobra.Command{

It then performs the following operations:
1. Get the latest closed PR matching the reference commit SHA.
2. Run gsutil to list, copy, and remove the vcr cassettes fixtures.
2. Run gcloud storage to list, copy, and remove the vcr cassettes fixtures.
`,
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
Expand Down Expand Up @@ -88,11 +88,12 @@ func mergeCassettes(basePath, baseBranch, prPath string, runner source.Runner) {

func listCassettes(path string, runner source.Runner) error {
lsArgs := []string{
"storage",
"ls",
path,
}
fmt.Println("Running command: ", "gsutil", lsArgs)
ret, err := runner.Run("gsutil", lsArgs, nil)
fmt.Println("Running command: ", "gcloud", lsArgs)
ret, err := runner.Run("gcloud", lsArgs, nil)
if err != nil {
return err
}
Expand All @@ -102,26 +103,26 @@ func listCassettes(path string, runner source.Runner) error {

func cpCassettes(src, dest string, runner source.Runner) {
cpArgs := []string{
"-m",
"storage",
"cp",
src,
dest,
}
fmt.Println("Running command: ", "gsutil", cpArgs)
if _, err := runner.Run("gsutil", cpArgs, nil); err != nil {
fmt.Println("Running command: ", "gcloud", cpArgs)
if _, err := runner.Run("gcloud", cpArgs, nil); err != nil {
fmt.Println("Error in copy: ", err)
}
}

func rmCassettes(dest string, runner source.Runner) {
rmArgs := []string{
"-m",
"storage",
"rm",
"-r",
"--recursive",
dest,
}
fmt.Println("Running command: ", "gsutil", rmArgs)
if _, err := runner.Run("gsutil", rmArgs, nil); err != nil {
fmt.Println("Running command: ", "gcloud", rmArgs)
if _, err := runner.Run("gcloud", rmArgs, nil); err != nil {
fmt.Println("Error in remove: ", err)
}
}
Expand Down
2 changes: 1 addition & 1 deletion .ci/magician/cmd/vcr_merge_eap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ var vcrMergeEapCmd = &cobra.Command{
1. CL number

It then performs the following operations:
1. Run gsutil to list, copy, and remove the vcr cassettes fixtures.
1. Run gcloud storage to list, copy, and remove the vcr cassettes fixtures.
`,
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
Expand Down
2 changes: 1 addition & 1 deletion mmv1/third_party/terraform/.teamcity/components/builds/build_steps.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ fun BuildSteps.saveArtifactsToGCS() {
fi

# Copy logs to GCS
gsutil -m cp %teamcity.build.checkoutDir%/debug* gs://teamcity-logs/${'$'}{FOLDER}/%env.BUILD_NUMBER%/
gcloud storage cp %teamcity.build.checkoutDir%/debug* gs://teamcity-logs/${'$'}{FOLDER}/%env.BUILD_NUMBER%/

# Cleanup
rm google-account.json
Expand Down
2 changes: 1 addition & 1 deletion mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2031,7 +2031,7 @@ resource "google_storage_bucket_object" "init_script" {
content = <<EOL
#!/bin/bash
echo "init action success" >> /tmp/%s
gsutil cp /tmp/%s ${google_storage_bucket.init_bucket.url}
gcloud storage cp /tmp/%s ${google_storage_bucket.init_bucket.url}
EOL

}
Expand Down
4 changes: 2 additions & 2 deletions ...d_party/terraform/services/storage/data_source_storage_object_signed_url_internal_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ const fakeCredentials = `{
}
`

// The following values are derived from the output of the `gsutil signurl` command.
// The following values are derived from the output of the `gcloud storage sign-url` command.
// i.e.
// gsutil signurl fake_creds.json gs://tf-test-bucket-6159205297736845881/path/to/file
// gcloud storage sign-url gs://tf-test-bucket-6159205297736845881/path/to/file --private-key-file=fake_creds.json
// URL HTTP Method Expiration Signed URL
// gs://tf-test-bucket-6159205297736845881/path/to/file GET 2016-08-12 14:03:30 https://storage.googleapis.com/tf-test-bucket-6159205297736845881/path/to/file?GoogleAccessId=user@gcp-project.iam.gserviceaccount.com&Expires=1470967410&Signature=JJvE2Jc%2BeoagyS1qRACKBGUkgLkKjw7cGymHhtB4IzzN3nbXDqr0acRWGy0%2BEpZ3HYNDalEYsK0lR9Q0WCgty5I0JKmPIuo9hOYa1xTNH%2B22xiWsekxGV%2FcA9FXgWpi%2BFt7fBmMk4dhDe%2BuuYc7N79hd0FYuSBNW1Wp32Bluoe4SNkNAB%2BuIDd9KqPzqs09UAbBoz2y4WxXOQnRyR8GAfb8B%2FDtv62gYjtmp%2F6%2Fyr6xj7byWKZdQt8kEftQLTQmP%2F17Efjp6p%2BXo71Q0F9IhAFiqWfp3Ij8hHDSebLcVb2ULXyHNNQpHBOhFgALrFW3I6Uc3WciLEOsBS9Ej3EGdTg%3D%3D

Expand Down
Loading