From 231a1728664daa7c8445197cfcdc89c1bbc2b3a3 Mon Sep 17 00:00:00 2001 From: Anis Elleuch Date: Fri, 22 Dec 2023 14:04:16 -0800 Subject: [PATCH] Separate tier add command to sub commands per tier backend Currently the flags help is crowded. This commit will split the add command to more sub-commands, minio, s3, gcs, azure; so it is easy for users to see associated flags for each backend tier --- cmd/admin-tier-deprecated.go | 2 +- cmd/ilm-tier-add-azure.go | 145 ++++++++++++++ cmd/ilm-tier-add-gcs.go | 106 +++++++++++ cmd/ilm-tier-add-minio.go | 120 ++++++++++++ cmd/ilm-tier-add-s3.go | 177 +++++++++++++++++ cmd/ilm-tier-add.go | 359 +++++------------------------------ 6 files changed, 593 insertions(+), 316 deletions(-) create mode 100644 cmd/ilm-tier-add-azure.go create mode 100644 cmd/ilm-tier-add-gcs.go create mode 100644 cmd/ilm-tier-add-minio.go create mode 100644 cmd/ilm-tier-add-s3.go diff --git a/cmd/admin-tier-deprecated.go b/cmd/admin-tier-deprecated.go index ae4f025acb..daf0926095 100644 --- a/cmd/admin-tier-deprecated.go +++ b/cmd/admin-tier-deprecated.go @@ -87,7 +87,7 @@ EXAMPLES: Hidden: true, OnUsageError: onUsageError, Before: setGlobalsFromContext, - Flags: append(globalFlags, adminTierAddFlags...), + Flags: append(globalFlags, adminTierAddMinioFlags...), CustomHelpTemplate: `NAME: {{.HelpName}} - {{.Usage}} diff --git a/cmd/ilm-tier-add-azure.go b/cmd/ilm-tier-add-azure.go new file mode 100644 index 0000000000..18854b202b --- /dev/null +++ b/cmd/ilm-tier-add-azure.go @@ -0,0 +1,145 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "github.com/minio/cli" + "github.com/minio/madmin-go/v3" + "github.com/minio/mc/pkg/probe" +) + +var adminTierAddAzureFlags = []cli.Flag{ + cli.StringFlag{ + Name: "endpoint", + Value: "", + Usage: "remote tier endpoint. e.g https://s3.amazonaws.com", + }, + cli.StringFlag{ + Name: "region", + Value: "", + Usage: "remote tier region. e.g us-west-2", + }, + cli.StringFlag{ + Name: "account-name", + Value: "", + Usage: "Azure Blob Storage account name", + }, + cli.StringFlag{ + Name: "account-key", + Value: "", + Usage: "Azure Blob Storage account key", + }, + cli.StringFlag{ + Name: "sp-tenant-id", + Value: "", + Usage: "Directory ID for the Azure service principal account", + }, + cli.StringFlag{ + Name: "sp-client-id", + Value: "", + Usage: "The client ID of the Azure service principal account", + }, + cli.StringFlag{ + Name: "sp-client-secret", + Value: "", + Usage: "The client secret of the Azure service principal account", + }, + cli.StringFlag{ + Name: "bucket", + Value: "", + Usage: "remote tier bucket", + }, + cli.StringFlag{ + Name: "prefix", + Value: "", + Usage: "remote tier prefix", + }, +} + +var adminTierAddAzureCmd = cli.Command{ + Name: "azure", + Usage: "add a new Azure remote tier target", + Action: mainAdminTierAddAzure, + OnUsageError: onUsageError, + Before: setGlobalsFromContext, + Flags: append(globalFlags, adminTierAddAzureFlags...), + CustomHelpTemplate: `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} ALIAS TIER-NAME [FLAGS] + +TIER-NAME: + Name of the remote tier target. e.g WARM-TIER + +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}} +EXAMPLES: + 1. Configure a new remote tier which transitions objects to a bucket in Azure Blob Storage: + {{.Prompt}} {{.HelpName}} azure myminio AZTIER --account-name ACCOUNT-NAME --account-key ACCOUNT-KEY \ + --bucket myazurebucket --prefix myazureprefix/ + +`, +} + +func fetchAzureTierConfig(ctx *cli.Context, tierName string) *madmin.TierConfig { + accountName := ctx.String("account-name") + accountKey := ctx.String("account-key") + if accountName == "" { + fatalIf(errDummy().Trace(), "Azure remote tier requires the storage account name") + } + + if accountKey == "" && (ctx.String("az-sp-tenant-id") == "" || ctx.String("az-sp-client-id") == "" || ctx.String("az-sp-client-secret") == "") { + fatalIf(errDummy().Trace(), "Azure remote tier requires static credentials OR service principal credentials") + } + + bucket := ctx.String("bucket") + if bucket == "" { + fatalIf(errDummy().Trace(), "Azure remote tier requires target bucket") + } + + azOpts := []madmin.AzureOptions{} + endpoint := ctx.String("endpoint") + if endpoint != "" { + azOpts = append(azOpts, madmin.AzureEndpoint(endpoint)) + } + + region := ctx.String("region") + if region != "" { + azOpts = append(azOpts, madmin.AzureRegion(region)) + } + + prefix := ctx.String("prefix") + if prefix != "" { + azOpts = append(azOpts, madmin.AzurePrefix(prefix)) + } + + if ctx.String("sp-tenant-id") != "" || ctx.String("sp-client-id") != "" || ctx.String("sp-client-secret") != "" { + azOpts = append(azOpts, madmin.AzureServicePrincipal(ctx.String("sp-tenant-id"), ctx.String("sp-client-id"), ctx.String("sp-client-secret"))) + } + + azCfg, e := madmin.NewTierAzure(tierName, accountName, accountKey, bucket, azOpts...) + fatalIf(probe.NewError(e), "Invalid configuration for Azure Blob Storage remote tier") + + return azCfg +} + +func mainAdminTierAddAzure(ctx *cli.Context) error { + return genericTierAddCmd(ctx, madmin.Azure) +} diff --git a/cmd/ilm-tier-add-gcs.go b/cmd/ilm-tier-add-gcs.go new file mode 100644 index 0000000000..eb6964975c --- /dev/null +++ b/cmd/ilm-tier-add-gcs.go @@ -0,0 +1,106 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "os" + + "github.com/minio/cli" + "github.com/minio/madmin-go/v3" + "github.com/minio/mc/pkg/probe" +) + +var adminTierAddGCSFlags = []cli.Flag{ + cli.StringFlag{ + Name: "region", + Value: "", + Usage: "remote tier region. e.g us-west-2", + }, + cli.StringFlag{ + Name: "credentials-file", + Value: "", + Usage: "path to Google Cloud Storage credentials file", + }, + cli.StringFlag{ + Name: "bucket", + Value: "", + Usage: "remote tier bucket", + }, + cli.StringFlag{ + Name: "prefix", + Value: "", + Usage: "remote tier prefix", + }, +} + +var adminTierAddGCSCmd = cli.Command{ + Name: "gcs", + Usage: "add a new GCS remote tier target", + Action: mainAdminTierAddGCS, + OnUsageError: onUsageError, + Before: setGlobalsFromContext, + Flags: append(globalFlags, adminTierAddGCSFlags...), + CustomHelpTemplate: `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} ALIAS TIER-NAME [FLAGS] + +TIER-NAME: + Name of the remote tier target. e.g WARM-TIER + +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}} +EXAMPLES: + 1. Configure a new remote tier which transitions objects to a bucket in Google Cloud Storage: + {{.Prompt}} {{.HelpName}} gcs myminio GCSTIER --credentials-file /path/to/credentials.json \ + --bucket mygcsbucket --prefix mygcsprefix/ +`, +} + +func fetchGCSTierConfig(ctx *cli.Context, tierName string) *madmin.TierConfig { + bucket := ctx.String("bucket") + if bucket == "" { + fatalIf(errInvalidArgument().Trace(), "GCS remote requires target bucket") + } + + gcsOpts := []madmin.GCSOptions{} + prefix := ctx.String("prefix") + if prefix != "" { + gcsOpts = append(gcsOpts, madmin.GCSPrefix(prefix)) + } + + region := ctx.String("region") + if region != "" { + gcsOpts = append(gcsOpts, madmin.GCSRegion(region)) + } + + credsPath := ctx.String("credentials-file") + credsBytes, e := os.ReadFile(credsPath) + fatalIf(probe.NewError(e), "Failed to read credentials file") + + gcsCfg, e := madmin.NewTierGCS(tierName, credsBytes, bucket, gcsOpts...) + fatalIf(probe.NewError(e), "Invalid configuration for Google Cloud Storage remote tier") + + return gcsCfg +} + +func mainAdminTierAddGCS(ctx *cli.Context) error { + return genericTierAddCmd(ctx, madmin.GCS) +} diff --git a/cmd/ilm-tier-add-minio.go b/cmd/ilm-tier-add-minio.go new file mode 100644 index 0000000000..b9b28665e3 --- /dev/null +++ b/cmd/ilm-tier-add-minio.go @@ -0,0 +1,120 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "github.com/minio/cli" + "github.com/minio/madmin-go/v3" + "github.com/minio/mc/pkg/probe" +) + +var adminTierAddMinioFlags = []cli.Flag{ + cli.StringFlag{ + Name: "endpoint", + Value: "", + Usage: "remote tier endpoint. e.g https://s3.amazonaws.com", + }, + cli.StringFlag{ + Name: "region", + Value: "", + Usage: "remote tier region. e.g us-west-2", + }, + cli.StringFlag{ + Name: "access-key", + Value: "", + Usage: "MinIO object storage access-key", + }, + cli.StringFlag{ + Name: "secret-key", + Value: "", + Usage: "MinIO object storage secret-key", + }, + cli.StringFlag{ + Name: "bucket", + Value: "", + Usage: "remote tier bucket", + }, + cli.StringFlag{ + Name: "prefix", + Value: "", + Usage: "remote tier prefix", + }, +} + +var adminTierAddMinioCmd = cli.Command{ + Name: "minio", + Usage: "add a new MinIO remote tier target", + Action: mainAdminTierAddMinio, + OnUsageError: onUsageError, + Before: setGlobalsFromContext, + Flags: append(globalFlags, adminTierAddMinioFlags...), + CustomHelpTemplate: `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} ALIAS TIER-NAME [FLAGS] + +TIER-NAME: + Name of the remote tier target. e.g WARM-TIER + +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}} +EXAMPLES: + 1. Configure a new remote tier which transitions objects to a bucket in a MinIO deployment: + {{.Prompt}} {{.HelpName}} minio myminio WARM-MINIO-TIER --endpoint https://warm-minio.com \ + --access-key ACCESSKEY --secret-key SECRETKEY --bucket mybucket --prefix myprefix/ +`, +} + +func fetchMinioTierConfig(ctx *cli.Context, tierName string) *madmin.TierConfig { + accessKey := ctx.String("access-key") + secretKey := ctx.String("secret-key") + if accessKey == "" || secretKey == "" { + fatalIf(errInvalidArgument().Trace(), "MinIO remote tier requires access credentials") + } + bucket := ctx.String("bucket") + if bucket == "" { + fatalIf(errInvalidArgument().Trace(), "MinIO remote tier requires target bucket") + } + + endpoint := ctx.String("endpoint") + if endpoint == "" { + fatalIf(errInvalidArgument().Trace(), "MinIO remote tier requires target endpoint") + } + + minioOpts := []madmin.MinIOOptions{} + prefix := ctx.String("prefix") + if prefix != "" { + minioOpts = append(minioOpts, madmin.MinIOPrefix(prefix)) + } + + region := ctx.String("region") + if region != "" { + minioOpts = append(minioOpts, madmin.MinIORegion(region)) + } + + minioCfg, e := madmin.NewTierMinIO(tierName, endpoint, accessKey, secretKey, bucket, minioOpts...) + fatalIf(probe.NewError(e), "Invalid configuration for MinIO tier") + + return minioCfg +} + +func mainAdminTierAddMinio(ctx *cli.Context) error { + return genericTierAddCmd(ctx, madmin.MinIO) +} diff --git a/cmd/ilm-tier-add-s3.go b/cmd/ilm-tier-add-s3.go new file mode 100644 index 0000000000..cfa9a11ee0 --- /dev/null +++ b/cmd/ilm-tier-add-s3.go @@ -0,0 +1,177 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + + "github.com/minio/cli" + "github.com/minio/madmin-go/v3" + "github.com/minio/mc/pkg/probe" +) + +var adminTierAddS3Flags = []cli.Flag{ + cli.StringFlag{ + Name: "endpoint", + Value: "", + Usage: "remote tier endpoint. e.g https://s3.amazonaws.com", + }, + cli.StringFlag{ + Name: "region", + Value: "", + Usage: "remote tier region. e.g us-west-2", + }, + cli.StringFlag{ + Name: "access-key", + Value: "", + Usage: "AWS S3 or compatible object storage access-key", + }, + cli.StringFlag{ + Name: "secret-key", + Value: "", + Usage: "AWS S3 or compatible object storage secret-key", + }, + cli.BoolFlag{ + Name: "use-aws-role", + Usage: "use AWS S3 role", + }, + cli.StringFlag{ + Name: "aws-role-arn", + Usage: "use AWS S3 role name", + }, + cli.StringFlag{ + Name: "aws-web-identity-file", + Usage: "use AWS S3 web identity file", + }, + cli.StringFlag{ + Name: "bucket", + Value: "", + Usage: "remote tier bucket", + }, + cli.StringFlag{ + Name: "prefix", + Value: "", + Usage: "remote tier prefix", + }, + cli.StringFlag{ + Name: "storage-class", + Value: "", + Usage: "remote tier storage-class", + }, +} + +var adminTierAddAWSCmd = cli.Command{ + Name: "s3", + Usage: "add a new AWS/S3 compatible remote tier target", + Action: mainAdminTierAddS3, + OnUsageError: onUsageError, + Before: setGlobalsFromContext, + Flags: append(globalFlags, adminTierAddS3Flags...), + CustomHelpTemplate: `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} ALIAS TIER-NAME [FLAGS] + +TIER-NAME: + Name of the remote tier target. e.g WARM-TIER + +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}} +EXAMPLES: + 1. Configure a new remote tier which transitions objects to a bucket in AWS S3 with STANDARD storage class: + {{.Prompt}} {{.HelpName}} s3 myminio S3TIER --endpoint https://s3.amazonaws.com \ + --access-key ACCESSKEY --secret-key SECRETKEY --bucket mys3bucket --prefix mys3prefix/ \ + --storage-class "STANDARD" --region us-west-2 +`, +} + +const ( + s3Standard = "STANDARD" + s3ReducedRedundancy = "REDUCED_REDUNDANCY" +) + +func fetchS3TierConfig(ctx *cli.Context, tierName string) *madmin.TierConfig { + accessKey := ctx.IsSet("access-key") + secretKey := ctx.IsSet("secret-key") + useAwsRole := ctx.IsSet("use-aws-role") + awsRoleArn := ctx.IsSet("aws-role-arn") + awsWebIdentity := ctx.IsSet("aws-web-identity-file") + + // Extensive flag check + switch { + case !accessKey && !secretKey && !useAwsRole && !awsRoleArn && !awsWebIdentity: + fatalIf(errInvalidArgument().Trace(), "No authentication mechanism was provided") + case (accessKey || secretKey) && (useAwsRole || awsRoleArn || awsWebIdentity): + fatalIf(errInvalidArgument().Trace(), "Static credentials cannot be combined with AWS role authentication") + case useAwsRole && (awsRoleArn || awsWebIdentity): + fatalIf(errInvalidArgument().Trace(), "--use-aws-role cannot be combined with --aws-role-arn or --aws-web-identity-file") + case (awsRoleArn && !awsWebIdentity) || (!awsRoleArn && awsWebIdentity): + fatalIf(errInvalidArgument().Trace(), "Both --use-aws-role and --aws-web-identity-file are required to enable web identity token based authentication") + case (accessKey && !secretKey) || (!accessKey && secretKey): + fatalIf(errInvalidArgument().Trace(), "Both --access-key and --secret-key are required to enable static credentials authentication") + + } + + bucket := ctx.String("bucket") + if bucket == "" { + fatalIf(errInvalidArgument().Trace(), "S3 remote tier requires target bucket") + } + + s3Opts := []madmin.S3Options{} + prefix := ctx.String("prefix") + if prefix != "" { + s3Opts = append(s3Opts, madmin.S3Prefix(prefix)) + } + + endpoint := ctx.String("endpoint") + if endpoint != "" { + s3Opts = append(s3Opts, madmin.S3Endpoint(endpoint)) + } + + region := ctx.String("region") + if region != "" { + s3Opts = append(s3Opts, madmin.S3Region(region)) + } + + s3SC := ctx.String("storage-class") + if s3SC != "" { + if s3SC != s3Standard && s3SC != s3ReducedRedundancy { + fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("unsupported storage-class type %s", s3SC)) + } + s3Opts = append(s3Opts, madmin.S3StorageClass(s3SC)) + } + if ctx.IsSet("use-aws-role") { + s3Opts = append(s3Opts, madmin.S3AWSRole()) + } + if ctx.IsSet("aws-role-arn") { + s3Opts = append(s3Opts, madmin.S3AWSRoleARN(ctx.String("aws-role-arn"))) + } + if ctx.IsSet("aws-web-identity-file") { + s3Opts = append(s3Opts, madmin.S3AWSRoleWebIdentityTokenFile(ctx.String("aws-web-identity-file"))) + } + s3Cfg, e := madmin.NewTierS3(tierName, ctx.String("access-key"), ctx.String("secret-key"), bucket, s3Opts...) + fatalIf(probe.NewError(e), "Invalid configuration for AWS S3 compatible remote tier") + + return s3Cfg +} + +func mainAdminTierAddS3(ctx *cli.Context) error { + return genericTierAddCmd(ctx, madmin.S3) +} diff --git a/cmd/ilm-tier-add.go b/cmd/ilm-tier-add.go index 383f0852ed..603bec3871 100644 --- a/cmd/ilm-tier-add.go +++ b/cmd/ilm-tier-add.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2022 MinIO, Inc. +// Copyright (c) 2015-2023 MinIO, Inc. // // This file is part of MinIO Object Storage stack // @@ -19,7 +19,6 @@ package cmd import ( "fmt" - "os" "strings" "github.com/fatih/color" @@ -30,317 +29,26 @@ import ( "github.com/minio/pkg/v2/console" ) -var adminTierAddFlags = []cli.Flag{ - cli.StringFlag{ - Name: "endpoint", - Value: "", - Usage: "remote tier endpoint. e.g https://s3.amazonaws.com", - }, - cli.StringFlag{ - Name: "region", - Value: "", - Usage: "remote tier region. e.g us-west-2", - }, - cli.StringFlag{ - Name: "access-key", - Value: "", - Usage: "AWS S3 or compatible object storage access-key", - }, - cli.StringFlag{ - Name: "secret-key", - Value: "", - Usage: "AWS S3 or compatible object storage secret-key", - }, - cli.BoolFlag{ - Name: "use-aws-role", - Usage: "use AWS S3 role", - }, - cli.StringFlag{ - Name: "aws-role-arn", - Usage: "use AWS S3 role name", - }, - cli.StringFlag{ - Name: "aws-web-identity-file", - Usage: "use AWS S3 web identity file", - }, - cli.StringFlag{ - Name: "account-name", - Value: "", - Usage: "Azure Blob Storage account name", - }, - cli.StringFlag{ - Name: "account-key", - Value: "", - Usage: "Azure Blob Storage account key", - }, - cli.StringFlag{ - Name: "az-sp-tenant-id", - Value: "", - Usage: "Directory ID for the Azure service principal account", - }, - cli.StringFlag{ - Name: "az-sp-client-id", - Value: "", - Usage: "The client ID of the Azure service principal account", - }, - cli.StringFlag{ - Name: "az-sp-client-secret", - Value: "", - Usage: "The client secret of the Azure service principal account", - }, - cli.StringFlag{ - Name: "credentials-file", - Value: "", - Usage: "path to Google Cloud Storage credentials file", - }, - cli.StringFlag{ - Name: "bucket", - Value: "", - Usage: "remote tier bucket", - }, - cli.StringFlag{ - Name: "prefix", - Value: "", - Usage: "remote tier prefix", - }, - cli.StringFlag{ - Name: "storage-class", - Value: "", - Usage: "remote tier storage-class", - }, - cli.BoolFlag{ - Name: "force", - Hidden: true, - Usage: "ignores in-use check for remote tier bucket/prefix", - }, +var ilmTierAddSubcommands = []cli.Command{ + adminTierAddAzureCmd, + adminTierAddGCSCmd, + adminTierAddAWSCmd, + adminTierAddMinioCmd, } var adminTierAddCmd = cli.Command{ - Name: "add", - Usage: "add a new remote tier target", - Action: mainAdminTierAdd, - OnUsageError: onUsageError, - Before: setGlobalsFromContext, - Flags: append(globalFlags, adminTierAddFlags...), - CustomHelpTemplate: `NAME: - {{.HelpName}} - {{.Usage}} - -USAGE: - {{.HelpName}} TYPE ALIAS NAME [FLAGS] - -TYPE: - Transition objects to supported cloud storage backend tier. Supported values are minio, s3, azure and gcs. - -NAME: - Name of the remote tier target. e.g WARM-TIER - -FLAGS: - {{range .VisibleFlags}}{{.}} - {{end}} -EXAMPLES: - 1. Configure a new remote tier which transitions objects to a bucket in a MinIO deployment: - {{.Prompt}} {{.HelpName}} minio myminio WARM-MINIO-TIER --endpoint https://warm-minio.com \ - --access-key ACCESSKEY --secret-key SECRETKEY --bucket mybucket --prefix myprefix/ - - 2. Configure a new remote tier which transitions objects to a bucket in Azure Blob Storage: - {{.Prompt}} {{.HelpName}} azure myminio AZTIER --account-name ACCOUNT-NAME --account-key ACCOUNT-KEY \ - --bucket myazurebucket --prefix myazureprefix/ - - 3. Configure a new remote tier which transitions objects to a bucket in AWS S3 with STANDARD storage class: - {{.Prompt}} {{.HelpName}} s3 myminio S3TIER --endpoint https://s3.amazonaws.com \ - --access-key ACCESSKEY --secret-key SECRETKEY --bucket mys3bucket --prefix mys3prefix/ \ - --storage-class "STANDARD" --region us-west-2 - - 4. Configure a new remote tier which transitions objects to a bucket in Google Cloud Storage: - {{.Prompt}} {{.HelpName}} gcs myminio GCSTIER --credentials-file /path/to/credentials.json \ - --bucket mygcsbucket --prefix mygcsprefix/ -`, -} - -// checkAdminTierAddSyntax validates all the positional arguments -func checkAdminTierAddSyntax(ctx *cli.Context) { - argsNr := len(ctx.Args()) - if argsNr < 3 { - showCommandHelpAndExit(ctx, 1) // last argument is exit code - } - if argsNr > 3 { - fatalIf(errInvalidArgument().Trace(ctx.Args().Tail()...), - "Incorrect number of arguments for tier add command.") - } + Name: "add", + Usage: "add a new remote tier target", + Action: mainAdminTierAdd, + OnUsageError: onUsageError, + Flags: globalFlags, + Before: setGlobalsFromContext, + HideHelpCommand: true, + Subcommands: ilmTierAddSubcommands, } -const ( - s3Standard = "STANDARD" - s3ReducedRedundancy = "REDUCED_REDUNDANCY" -) - -// fetchTierConfig returns a TierConfig given a tierName, a tierType and ctx to -// lookup command-line flags from. It exits with non-zero error code if any of -// the flags contain invalid values. -func fetchTierConfig(ctx *cli.Context, tierName string, tierType madmin.TierType) *madmin.TierConfig { - switch tierType { - case madmin.MinIO: - accessKey := ctx.String("access-key") - secretKey := ctx.String("secret-key") - if accessKey == "" || secretKey == "" { - fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires access credentials", tierType)) - } - bucket := ctx.String("bucket") - if bucket == "" { - fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires target bucket", tierType)) - } - - endpoint := ctx.String("endpoint") - if endpoint == "" { - fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires target endpoint", tierType)) - } - - minioOpts := []madmin.MinIOOptions{} - prefix := ctx.String("prefix") - if prefix != "" { - minioOpts = append(minioOpts, madmin.MinIOPrefix(prefix)) - } - - region := ctx.String("region") - if region != "" { - minioOpts = append(minioOpts, madmin.MinIORegion(region)) - } - - minioCfg, e := madmin.NewTierMinIO(tierName, endpoint, accessKey, secretKey, bucket, minioOpts...) - fatalIf(probe.NewError(e), "Invalid configuration for MinIO tier") - - return minioCfg - - case madmin.S3: - accessKey := ctx.IsSet("access-key") - secretKey := ctx.IsSet("secret-key") - useAwsRole := ctx.IsSet("use-aws-role") - awsRoleArn := ctx.IsSet("aws-role-arn") - awsWebIdentity := ctx.IsSet("aws-web-identity-file") - - // Extensive flag check - switch { - case !accessKey && !secretKey && !useAwsRole && !awsRoleArn && !awsWebIdentity: - fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s: No authentication mechanism was provided", tierType)) - case (accessKey || secretKey) && (useAwsRole || awsRoleArn || awsWebIdentity): - fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s: Static credentials cannot be combined with AWS role authentication", tierType)) - case useAwsRole && (awsRoleArn || awsWebIdentity): - fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s: --use-aws-role cannot be combined with --aws-role-arn or --aws-web-identity-file", tierType)) - case (awsRoleArn && !awsWebIdentity) || (!awsRoleArn && awsWebIdentity): - fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s: Both --use-aws-role and --aws-web-identity-file are required to enable web identity token based authentication", tierType)) - case (accessKey && !secretKey) || (!accessKey && secretKey): - fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s: Both --access-key and --secret-key are required to enable static credentials authentication", tierType)) - - } - - bucket := ctx.String("bucket") - if bucket == "" { - fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires target bucket", tierType)) - } - - s3Opts := []madmin.S3Options{} - prefix := ctx.String("prefix") - if prefix != "" { - s3Opts = append(s3Opts, madmin.S3Prefix(prefix)) - } - - endpoint := ctx.String("endpoint") - if endpoint != "" { - s3Opts = append(s3Opts, madmin.S3Endpoint(endpoint)) - } - - region := ctx.String("region") - if region != "" { - s3Opts = append(s3Opts, madmin.S3Region(region)) - } - - s3SC := ctx.String("storage-class") - if s3SC != "" { - if s3SC != s3Standard && s3SC != s3ReducedRedundancy { - fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("unsupported storage-class type %s", s3SC)) - } - s3Opts = append(s3Opts, madmin.S3StorageClass(s3SC)) - } - if ctx.IsSet("use-aws-role") { - s3Opts = append(s3Opts, madmin.S3AWSRole()) - } - if ctx.IsSet("aws-role-arn") { - s3Opts = append(s3Opts, madmin.S3AWSRoleARN(ctx.String("aws-role-arn"))) - } - if ctx.IsSet("aws-web-identity-file") { - s3Opts = append(s3Opts, madmin.S3AWSRoleWebIdentityTokenFile(ctx.String("aws-web-identity-file"))) - } - s3Cfg, e := madmin.NewTierS3(tierName, ctx.String("access-key"), ctx.String("secret-key"), bucket, s3Opts...) - fatalIf(probe.NewError(e), "Invalid configuration for AWS S3 compatible remote tier") - - return s3Cfg - case madmin.Azure: - accountName := ctx.String("account-name") - accountKey := ctx.String("account-key") - if accountName == "" { - fatalIf(errDummy().Trace(), fmt.Sprintf("%s remote tier requires the storage account name", tierType)) - } - - if accountKey == "" && (ctx.String("az-sp-tenant-id") == "" || ctx.String("az-sp-client-id") == "" || ctx.String("az-sp-client-secret") == "") { - fatalIf(errDummy().Trace(), fmt.Sprintf("%s remote tier requires static credentials OR service principal credentials", tierType)) - } - - bucket := ctx.String("bucket") - if bucket == "" { - fatalIf(errDummy().Trace(), fmt.Sprintf("%s remote tier requires target bucket", tierType)) - } - - azOpts := []madmin.AzureOptions{} - endpoint := ctx.String("endpoint") - if endpoint != "" { - azOpts = append(azOpts, madmin.AzureEndpoint(endpoint)) - } - - region := ctx.String("region") - if region != "" { - azOpts = append(azOpts, madmin.AzureRegion(region)) - } - - prefix := ctx.String("prefix") - if prefix != "" { - azOpts = append(azOpts, madmin.AzurePrefix(prefix)) - } - - if ctx.String("az-sp-tenant-id") != "" || ctx.String("az-sp-client-id") != "" || ctx.String("az-sp-client-secret") != "" { - azOpts = append(azOpts, madmin.AzureServicePrincipal(ctx.String("az-sp-tenant-id"), ctx.String("az-sp-client-id"), ctx.String("az-sp-client-secret"))) - } - - azCfg, e := madmin.NewTierAzure(tierName, accountName, accountKey, bucket, azOpts...) - fatalIf(probe.NewError(e), "Invalid configuration for Azure Blob Storage remote tier") - - return azCfg - case madmin.GCS: - bucket := ctx.String("bucket") - if bucket == "" { - fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote requires target bucket", tierType)) - } - - gcsOpts := []madmin.GCSOptions{} - prefix := ctx.String("prefix") - if prefix != "" { - gcsOpts = append(gcsOpts, madmin.GCSPrefix(prefix)) - } - - region := ctx.String("region") - if region != "" { - gcsOpts = append(gcsOpts, madmin.GCSRegion(region)) - } - - credsPath := ctx.String("credentials-file") - credsBytes, e := os.ReadFile(credsPath) - fatalIf(probe.NewError(e), "Failed to read credentials file") - - gcsCfg, e := madmin.NewTierGCS(tierName, credsBytes, bucket, gcsOpts...) - fatalIf(probe.NewError(e), "Invalid configuration for Google Cloud Storage remote tier") - - return gcsCfg - } - fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("Invalid remote tier type %s", tierType)) +func mainAdminTierAdd(ctx *cli.Context) error { + commandNotFound(ctx, ilmTierAddSubcommands) return nil } @@ -402,18 +110,27 @@ func (msg *tierMessage) SetTierConfig(sCfg *madmin.TierConfig) { } } -func mainAdminTierAdd(ctx *cli.Context) error { +// checkAdminTierAddSyntax validates all the positional arguments +func checkAdminTierAddSyntax(ctx *cli.Context) { + argsNr := len(ctx.Args()) + if argsNr < 2 { + showCommandHelpAndExit(ctx, 1) // last argument is exit code + } + if argsNr > 2 { + fatalIf(errInvalidArgument().Trace(ctx.Args().Tail()...), + "Incorrect number of arguments for tier add command.") + } +} + +func genericTierAddCmd(ctx *cli.Context, tierType madmin.TierType) error { checkAdminTierAddSyntax(ctx) console.SetColor("TierMessage", color.New(color.FgGreen)) args := ctx.Args() - tierTypeStr := args.Get(0) - tierType, e := madmin.NewTierType(tierTypeStr) - fatalIf(probe.NewError(e), "Unsupported tier type") - aliasedURL := args.Get(1) - tierName := args.Get(2) + aliasedURL := args.Get(0) + tierName := strings.ToUpper(args.Get(1)) if tierName == "" { fatalIf(errInvalidArgument(), "Tier name can't be empty") } @@ -422,7 +139,19 @@ func mainAdminTierAdd(ctx *cli.Context) error { client, cerr := newAdminClient(aliasedURL) fatalIf(cerr, "Unable to initialize admin connection.") - tCfg := fetchTierConfig(ctx, strings.ToUpper(tierName), tierType) + var tCfg *madmin.TierConfig + + switch tierType { + case madmin.MinIO: + tCfg = fetchMinioTierConfig(ctx, tierName) + case madmin.S3: + tCfg = fetchS3TierConfig(ctx, tierName) + case madmin.Azure: + tCfg = fetchAzureTierConfig(ctx, tierName) + case madmin.GCS: + tCfg = fetchGCSTierConfig(ctx, tierName) + } + ignoreInUse := ctx.Bool("force") if ignoreInUse { fatalIf(probe.NewError(client.AddTierIgnoreInUse(globalContext, tCfg)).Trace(args...), "Unable to configure remote tier target")