list very large buckets using "bucket inventory"

* part seven, prev. commit: 8736c91 Signed-off-by: Alex Aizman <[email protected]>
NVIDIA · Mar 11, 2024 · fb8096b · fb8096b
1 parent 474c875
commit fb8096b
Show file tree

Hide file tree

Showing 13 changed files with 39 additions and 26 deletions.
diff --git a/ais/backend/ais.go b/ais/backend/ais.go
@@ -5,6 +5,7 @@
 package backend
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"net/http"
@@ -391,7 +392,7 @@ func (*AISBackendProvider) CreateBucket(_ *meta.Bck) (errCode int, err error) {
 // that, in particular, include `dontAddRemote` = (true | false).
 // Here we have to hardcode the value to keep HeadBucket() consistent across all backends.
 // For similar limitations, see also ListBuckets() below.
-func (m *AISBackendProvider) HeadBucket(_ ctx, remoteBck *meta.Bck) (bckProps cos.StrKVs, errCode int, err error) {
+func (m *AISBackendProvider) HeadBucket(_ context.Context, remoteBck *meta.Bck) (bckProps cos.StrKVs, errCode int, err error) {
 	var (
 		remAis      *remAis
 		p           *cmn.Bprops
@@ -519,7 +520,7 @@ func (m *AISBackendProvider) blist(uuid string, qbck cmn.QueryBcks) (bcks cmn.Bc
 // in part including apc.Flt* location specifier.
 // Here, and elsewhere down below, we hardcode (the default) `apc.FltPresent` to, eesentially,
 // keep HeadObj() consistent across backends.
-func (m *AISBackendProvider) HeadObj(_ ctx, lom *core.LOM) (oa *cmn.ObjAttrs, errCode int, err error) {
+func (m *AISBackendProvider) HeadObj(_ context.Context, lom *core.LOM) (oa *cmn.ObjAttrs, errCode int, err error) {
 	var (
 		remAis    *remAis
 		op        *cmn.ObjectProps
@@ -539,7 +540,7 @@ func (m *AISBackendProvider) HeadObj(_ ctx, lom *core.LOM) (oa *cmn.ObjAttrs, er
 	return
 }
 
-func (m *AISBackendProvider) GetObj(_ ctx, lom *core.LOM, owt cmn.OWT) (errCode int, err error) {
+func (m *AISBackendProvider) GetObj(_ context.Context, lom *core.LOM, owt cmn.OWT) (errCode int, err error) {
 	var (
 		remAis    *remAis
 		r         io.ReadCloser
@@ -566,7 +567,7 @@ func (m *AISBackendProvider) GetObj(_ ctx, lom *core.LOM, owt cmn.OWT) (errCode
 	return extractErrCode(err, remAis.uuid)
 }
 
-func (m *AISBackendProvider) GetObjReader(_ ctx, lom *core.LOM, offset, length int64) (res core.GetReaderResult) {
+func (m *AISBackendProvider) GetObjReader(_ context.Context, lom *core.LOM, offset, length int64) (res core.GetReaderResult) {
 	var (
 		remAis    *remAis
 		op        *cmn.ObjectProps

diff --git a/ais/backend/aws.go b/ais/backend/aws.go
@@ -84,7 +84,7 @@ func (*awsProvider) CreateBucket(_ *meta.Bck) (int, error) {
 // HEAD BUCKET
 //
 
-func (*awsProvider) HeadBucket(_ ctx, bck *meta.Bck) (bckProps cos.StrKVs, errCode int, err error) {
+func (*awsProvider) HeadBucket(_ context.Context, bck *meta.Bck) (bckProps cos.StrKVs, errCode int, err error) {
 	var (
 		svc      *s3.Client
 		region   string
@@ -325,7 +325,7 @@ func (*awsProvider) ListBuckets(cmn.QueryBcks) (bcks cmn.Bcks, errCode int, err
 // HEAD OBJECT
 //
 
-func (*awsProvider) HeadObj(_ ctx, lom *core.LOM) (oa *cmn.ObjAttrs, errCode int, err error) {
+func (*awsProvider) HeadObj(_ context.Context, lom *core.LOM) (oa *cmn.ObjAttrs, errCode int, err error) {
 	var (
 		svc        *s3.Client
 		headOutput *s3.HeadObjectOutput

diff --git a/ais/backend/common.go b/ais/backend/common.go
@@ -5,7 +5,6 @@
 package backend
 
 import (
-	"context"
 	"time"
 
 	"github.com/NVIDIA/aistore/cmn"
@@ -14,8 +13,6 @@ import (
 	"github.com/NVIDIA/aistore/fs"
 )
 
-type ctx = context.Context // used when omitted for shortness sake
-
 func fmtTime(t time.Time) string { return t.Format(time.RFC3339) }
 
 func calcPageSize(pageSize, maxPageSize int64) int64 {

diff --git a/ais/backend/hdfs.go b/ais/backend/hdfs.go
@@ -109,7 +109,7 @@ func (hp *hdfsProvider) checkDirectoryExists(bck *meta.Bck) (errCode int, err er
 // HEAD BUCKET
 //
 
-func (hp *hdfsProvider) HeadBucket(_ ctx, bck *meta.Bck) (bckProps cos.StrKVs,
+func (hp *hdfsProvider) HeadBucket(_ context.Context, bck *meta.Bck) (bckProps cos.StrKVs,
 	errCode int, err error) {
 	if errCode, err = hp.checkDirectoryExists(bck); err != nil {
 		return
@@ -224,7 +224,7 @@ func (*hdfsProvider) ListBuckets(cmn.QueryBcks) (buckets cmn.Bcks, errCode int,
 // HEAD OBJECT
 //
 
-func (hp *hdfsProvider) HeadObj(_ ctx, lom *core.LOM) (oa *cmn.ObjAttrs, errCode int, err error) {
+func (hp *hdfsProvider) HeadObj(_ context.Context, lom *core.LOM) (oa *cmn.ObjAttrs, errCode int, err error) {
 	var (
 		fr       *hdfs.FileReader
 		filePath = filepath.Join(lom.Bck().Props.Extra.HDFS.RefDirectory, lom.ObjName)

diff --git a/ais/backend/mock_backend.go b/ais/backend/mock_backend.go
@@ -35,7 +35,7 @@ func (*mockBP) CreateBucket(*meta.Bck) (int, error) {
 	return http.StatusBadRequest, cmn.NewErrUnsupp("create", mock+" bucket")
 }
 
-func (*mockBP) HeadBucket(_ ctx, bck *meta.Bck) (cos.StrKVs, int, error) {
+func (*mockBP) HeadBucket(_ context.Context, bck *meta.Bck) (cos.StrKVs, int, error) {
 	return cos.StrKVs{}, http.StatusNotFound, cmn.NewErrRemoteBckOffline(bck.Bucket())
 }
 
@@ -52,11 +52,11 @@ func (*mockBP) ListBuckets(cmn.QueryBcks) (bcks cmn.Bcks, errCode int, err error
 	return
 }
 
-func (*mockBP) HeadObj(_ ctx, lom *core.LOM) (*cmn.ObjAttrs, int, error) {
+func (*mockBP) HeadObj(_ context.Context, lom *core.LOM) (*cmn.ObjAttrs, int, error) {
 	return &cmn.ObjAttrs{}, http.StatusNotFound, cmn.NewErrRemoteBckNotFound(lom.Bucket())
 }
 
-func (*mockBP) GetObj(_ ctx, lom *core.LOM, _ cmn.OWT) (int, error) {
+func (*mockBP) GetObj(_ context.Context, lom *core.LOM, _ cmn.OWT) (int, error) {
 	return http.StatusNotFound, cmn.NewErrRemoteBckNotFound(lom.Bucket())
 }
 

diff --git a/ais/proxy.go b/ais/proxy.go
@@ -2244,9 +2244,6 @@ func (p *proxy) lsObjsR(bck *meta.Bck, lsmsg *apc.LsoMsg, hdr http.Header, smap
 	// Combine the results.
 	resLists := make([]*cmn.LsoResult, 0, len(results))
 	for _, res := range results {
-		if res.status == http.StatusNotFound {
-			continue
-		}
 		if res.err != nil {
 			err := res.toErr()
 			freeBcastRes(results)

diff --git a/cmd/cli/cli/bucket_hdlr.go b/cmd/cli/cli/bucket_hdlr.go
@@ -140,7 +140,10 @@ var (
 			silentFlag,
 			dontWaitFlag,
 			verChangedFlag,
+			// bucket inventory
 			useInventoryFlag,
+			invNameFlag,
+			invIDFlag,
 		},
 
 		cmdLRU: {

diff --git a/cmd/cli/cli/const.go b/cmd/cli/cli/const.go
@@ -527,10 +527,19 @@ var (
 			indent4 + "\t- applies to remote backends that maintain at least some form of versioning information (e.g., version, checksum, ETag)\n" +
 			indent4 + "\t- see related: 'ais get --latest', 'ais cp --sync', 'ais prefetch --latest'",
 	}
+
 	useInventoryFlag = cli.BoolFlag{
 		Name:  "inventory",
 		Usage: "experimental; requires s3:// backend",
 	}
+	invNameFlag = cli.StringFlag{
+		Name:  "inv-name", // compare w/ HdrInvName
+		Usage: "bucket inventory name (optional; system default name is '.inventory')",
+	}
+	invIDFlag = cli.StringFlag{
+		Name:  "inv-id", // cpmpare w/ HdrInvID
+		Usage: "bucket inventory ID (optional; by default, we use bucket name as the bucket's inventory ID)",
+	}
 
 	keepMDFlag       = cli.BoolFlag{Name: "keep-md", Usage: "keep bucket metadata"}
 	dataSlicesFlag   = cli.IntFlag{Name: "data-slices,data,d", Usage: "number of data slices", Required: true}

diff --git a/cmd/cli/cli/get.go b/cmd/cli/cli/get.go
@@ -204,7 +204,8 @@ func getMultiObj(c *cli.Context, bck cmn.Bck, archpath, outFile string, extract
 	if flagIsSet(c, useInventoryFlag) {
 		lsargs.Header = http.Header{
 			apc.HdrInventory: []string{"true"},
-			apc.HdrInvName:   []string{"inv-all"}, // TODO -- FIXME: remove; provide via flag
+			apc.HdrInvName:   []string{parseStrFlag(c, invNameFlag)},
+			apc.HdrInvID:     []string{parseStrFlag(c, invIDFlag)},
 		}
 	}
 

diff --git a/cmd/cli/cli/ls.go b/cmd/cli/cli/ls.go
@@ -376,7 +376,8 @@ func listObjects(c *cli.Context, bck cmn.Bck, prefix string, listArch bool) erro
 	if flagIsSet(c, useInventoryFlag) {
 		lsargs.Header = http.Header{
 			apc.HdrInventory: []string{"true"},
-			apc.HdrInvName:   []string{"inv-all"}, // TODO -- FIXME: remove; provide via flag
+			apc.HdrInvName:   []string{parseStrFlag(c, invNameFlag)},
+			apc.HdrInvID:     []string{parseStrFlag(c, invIDFlag)},
 		}
 	}
 

diff --git a/cmd/cli/cli/object_hdlr.go b/cmd/cli/cli/object_hdlr.go
@@ -48,15 +48,19 @@ var (
 			// archive
 			archpathGetFlag,
 			extractFlag,
+			// bucket inventory
+			useInventoryFlag,
+			invNameFlag,
+			invIDFlag,
 			// multi-object options (passed to list-objects)
 			getObjPrefixFlag,
 			getObjCachedFlag,
 			listArchFlag,
 			objLimitFlag,
-			unitsFlag,
-			useInventoryFlag, // experimental
-			verboseFlag,      // client side
-			silentFlag,       // server side
+			//
+			unitsFlag,   // raw (bytes), kb, mib, etc.
+			verboseFlag, // client side
+			silentFlag,  // server side
 		},
 
 		commandPut: append(

diff --git a/cmd/cli/go.mod b/cmd/cli/go.mod
@@ -3,7 +3,7 @@ module github.com/NVIDIA/aistore/cmd/cli
 go 1.22
 
 require (
-	github.com/NVIDIA/aistore v1.3.23-0.20240308185015-d250511d14a5
+	github.com/NVIDIA/aistore v1.3.23-0.20240311180400-474c875fa5cd
 	github.com/fatih/color v1.16.0
 	github.com/json-iterator/go v1.1.12
 	github.com/onsi/ginkgo v1.16.5

diff --git a/cmd/cli/go.sum b/cmd/cli/go.sum
@@ -1,7 +1,7 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/NVIDIA/aistore v1.3.23-0.20240308185015-d250511d14a5 h1:J3bZmSGL5/IPPxyWSoyoi6fqxp1G7zJW6SCq0hGHF/4=
-github.com/NVIDIA/aistore v1.3.23-0.20240308185015-d250511d14a5/go.mod h1:V9DB29cNiY0k7bd5f5uBBKprzMiyk4ZdR4wdIsFjt3E=
+github.com/NVIDIA/aistore v1.3.23-0.20240311180400-474c875fa5cd h1:s+aFAru1kLlyJR/TmIh/eJ/e25UZcrGbVaSabkczNZQ=
+github.com/NVIDIA/aistore v1.3.23-0.20240311180400-474c875fa5cd/go.mod h1:V9DB29cNiY0k7bd5f5uBBKprzMiyk4ZdR4wdIsFjt3E=
 github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
 github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
 github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=