From 4d75c7d65bcb967f1260d16156b1dd7c57614a99 Mon Sep 17 00:00:00 2001 From: Alex Aizman Date: Fri, 10 Jan 2025 12:58:49 -0500 Subject: [PATCH] core: remove "file name too long" limitation; add test * part five, prev. commit: d22197625e9228 Signed-off-by: Alex Aizman --- ais/test/common_test.go | 2 ++ ais/test/smoke_test.go | 14 +++++++++----- cmd/cli/cli/multiobj.go | 6 ++++-- cmd/cli/cli/parse_uri.go | 4 ++++ core/lcopy.go | 9 +-------- core/lfile.go | 13 +++++-------- core/lom.go | 40 ++++++++++++++++++++++++++++------------ fs/content.go | 18 +++++++++++++----- tools/client.go | 4 +++- 9 files changed, 69 insertions(+), 41 deletions(-) diff --git a/ais/test/common_test.go b/ais/test/common_test.go index 1f7a6f3a283..558ba813199 100644 --- a/ais/test/common_test.go +++ b/ais/test/common_test.go @@ -68,6 +68,7 @@ type ioContext struct { originalProxyCount int num int numGetsEachFile int + nameLen int getErrIsFatal bool silent bool fixedSize bool @@ -232,6 +233,7 @@ func (m *ioContext) puts(ignoreErrs ...bool) { Bck: m.bck, ObjPath: m.prefix, ObjCnt: m.num, + ObjNameLn: m.nameLen, ObjSize: m.fileSize, FixedSize: m.fixedSize, CksumType: p.Cksum.Type, diff --git a/ais/test/smoke_test.go b/ais/test/smoke_test.go index 1dbf0afbc21..fd9a95b3ea6 100644 --- a/ais/test/smoke_test.go +++ b/ais/test/smoke_test.go @@ -1,6 +1,6 @@ // Package integration_test. /* - * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved. */ package integration_test @@ -11,17 +11,21 @@ import ( "github.com/NVIDIA/aistore/core/meta" ) -func TestSmoke(t *testing.T) { - objSizes := [3]uint64{3 * cos.KiB, 19 * cos.KiB, 77 * cos.KiB} +func TestSmoke(t *testing.T) { _wrd(t, 0) } +func TestFntl(t *testing.T) { _wrd(t, 280) } + +func _wrd(t *testing.T, nameLen int) { + objSizes := [...]uint64{3 * cos.KiB, 19 * cos.KiB, 77 * cos.KiB} runProviderTests(t, func(t *testing.T, bck *meta.Bck) { for _, objSize := range objSizes { - name := "size:" + cos.ToSizeIEC(int64(objSize), 0) - t.Run(name, func(t *testing.T) { + tname := "size:" + cos.ToSizeIEC(int64(objSize), 0) + t.Run(tname, func(t *testing.T) { m := ioContext{ t: t, bck: bck.Clone(), num: 100, + nameLen: nameLen, fileSize: objSize, prefix: "smoke/obj-", } diff --git a/cmd/cli/cli/multiobj.go b/cmd/cli/cli/multiobj.go index dcaceca1429..a98b110a669 100644 --- a/cmd/cli/cli/multiobj.go +++ b/cmd/cli/cli/multiobj.go @@ -232,9 +232,11 @@ func _rmOne(c *cli.Context, shift int) error { return err } if shouldHeadRemote(c, bck) { - if _, err := headBucket(bck, false /* don't add */); err != nil { + bprops, err := headBucket(bck, false /* don't add */) + if err != nil { return err } + bck.Props = bprops } // [NOTE] // - passing empty bck _not_ to interpret embedded objName as prefix @@ -262,7 +264,7 @@ func _rmOne(c *cli.Context, shift int) error { qflprn(listFlag), qflprn(templateFlag), qflprn(rmrfFlag)) default: // 3. one obj err := api.DeleteObject(apiBP, bck, oltp.objName) - if err == nil && oltp.notFound { + if err == nil && bck.IsCloud() && oltp.notFound { // [NOTE] // - certain backends return OK when specified object does not exist (see aws.go) // - compensate here diff --git a/cmd/cli/cli/parse_uri.go b/cmd/cli/cli/parse_uri.go index acfb1c08c8c..4110dbf8fb7 100644 --- a/cmd/cli/cli/parse_uri.go +++ b/cmd/cli/cli/parse_uri.go @@ -266,6 +266,10 @@ func dopOLTP(c *cli.Context, bck cmn.Bck, objNameOrTmpl string) (oltp oltp, err case flagIsSet(c, noRecursFlag): oltp.objName = objNameOrTmpl + case len(objNameOrTmpl) > 255: + // not running lsObjVsPref when "prefix-or-objname" is that long + oltp.objName = objNameOrTmpl + default: // [NOTE] additional list-objects call to disambiguate: differentiate embedded prefix from object name dop, err := lsObjVsPref(bck, objNameOrTmpl) diff --git a/core/lcopy.go b/core/lcopy.go index bbd8ffc79ee..18249a131a8 100644 --- a/core/lcopy.go +++ b/core/lcopy.go @@ -239,16 +239,9 @@ func (lom *LOM) Copy(mi *fs.Mountpath, buf []byte) (err error) { } } } - } else if cos.IsErrFntl(errExists) { - // fixup fntl - short := lom.ShortenFntl() - saved := lom.PushFntl(short) - defer lom.PopFntl(saved) - copyFQN = mi.MakePathFQN(lom.Bucket(), fs.ObjectType, lom.ObjName) - workFQN = mi.MakePathFQN(lom.Bucket(), fs.WorkfileType, fs.WorkfileCopy+"."+lom.ObjName) } - // copy + // do _, _, err = cos.CopyFile(lom.FQN, workFQN, buf, cos.ChecksumNone) // TODO: checksumming if err != nil { return diff --git a/core/lfile.go b/core/lfile.go index 13552d79f66..38bc464ffb6 100644 --- a/core/lfile.go +++ b/core/lfile.go @@ -8,13 +8,11 @@ import ( "fmt" "os" "path/filepath" - rdebug "runtime/debug" "syscall" "github.com/NVIDIA/aistore/cmn" "github.com/NVIDIA/aistore/cmn/cos" "github.com/NVIDIA/aistore/cmn/debug" - "github.com/NVIDIA/aistore/cmn/nlog" ) const ( @@ -48,7 +46,7 @@ func (lom *LOM) OpenFile() (fh *os.File, _ error) { return fh, nil } -// same as above but return reader +// same as above but return a reader func (lom *LOM) Open() (fh cos.LomReader, err error) { fh, err = os.Open(lom.FQN) switch { @@ -59,12 +57,10 @@ func (lom *LOM) Open() (fh cos.LomReader, err error) { err = e } return nil, err + + // case cos.IsErrFntl(err) + default: - // DEBUG - if cos.IsErrFntl(err) { - nlog.Errorln(">>>", err) - rdebug.PrintStack() - } return nil, err } } @@ -149,6 +145,7 @@ func (*LOM) AppendWork(wfqn string) (fh cos.LomWriter, err error) { func (lom *LOM) RemoveMain() error { return cos.RemoveFile(lom.FQN) + // if err != nil && cos.IsErrFntl(err) } func (lom *LOM) RemoveObj(force ...bool) (err error) { diff --git a/core/lom.go b/core/lom.go index 2fb1e984cb5..23f2414fd42 100644 --- a/core/lom.go +++ b/core/lom.go @@ -417,6 +417,9 @@ func (lom *LOM) Load(cacheit, locked bool) error { // fast path if lmd != nil { lom.md = *lmd + if lom.IsFntl() { + lom.fixupFntl() + } return lom._checkBucket(bmd) } @@ -427,15 +430,15 @@ func (lom *LOM) Load(cacheit, locked bool) error { if err := lom.FromFS(); err != nil { return err } - if lom.bid() == 0 { - // NOTE: always zero (MetaverLOM = 1: not storing the BID part of the lom.md.lid) - lom.setbid(lom.Bprops().BID) - } + + // MetaverLOM = 1: always zero (not storing lom.md.lid) + debug.Assert(lom.bid() == 0 || lom.bid() == lom.Bprops().BID, lom.bid()) + lom.setbid(lom.Bprops().BID) if err := lom._checkBucket(bmd); err != nil { return err } - if cacheit && lcache != nil { + if cacheit { md := lom.md lcache.Store(lom.digest, &md) } @@ -467,16 +470,18 @@ func (lom *LOM) LoadUnsafe() (err error) { // fast path if lmd != nil { lom.md = *lmd + if lom.IsFntl() { + lom.fixupFntl() + } return lom._checkBucket(bmd) } // read and decode xattr; NOTE: fs.GetXattr* vs fs.SetXattr race possible and must be // either a) handled or b) benign from the caller's perspective if _, err = lom.lmfs(true); err == nil { - if lom.bid() == 0 { - // ditto MetaverLOM = 1 - lom.setbid(lom.Bprops().BID) - } + // MetaverLOM = 1: always zero (not storing lom.md.lid) + debug.Assert(lom.bid() == 0 || lom.bid() == lom.Bprops().BID, lom.bid()) + lom.setbid(lom.Bprops().BID) err = lom._checkBucket(bmd) } return err @@ -674,8 +679,10 @@ func (lom *LOM) HasFntlPrefix() bool { return strings.HasPrefix(lom.ObjName, fs.PrefixFntl) } +// (compare with fs/content Gen()) func (lom *LOM) ShortenFntl() []string { - // (compare with fs/content Gen()) + debug.Assert(fs.FnameTooLong(lom.ObjName), lom.FQN) + noname := fs.PrefixFntl + cos.ChecksumB2S(cos.UnsafeB(lom.FQN), cos.ChecksumSHA256) nfqn := lom.mi.MakePathFQN(lom.Bucket(), fs.ObjectType, noname) @@ -683,6 +690,15 @@ func (lom *LOM) ShortenFntl() []string { return []string{nfqn, noname} } +func (lom *LOM) fixupFntl() { + if !fs.FnameTooLong(lom.ObjName) { + return + } + lom.ObjName = fs.PrefixFntl + cos.ChecksumB2S(cos.UnsafeB(lom.FQN), cos.ChecksumSHA256) // noname + lom.FQN = lom.mi.MakePathFQN(lom.Bucket(), fs.ObjectType, lom.ObjName) // nfqn + lom.HrwFQN = &lom.FQN +} + func (lom *LOM) OrigFntl() []string { debug.Assert(lom.IsFntl()) ofqn, ok := lom.GetCustomKey(cmn.OrigFntl) @@ -698,9 +714,9 @@ func (lom *LOM) OrigFntl() []string { return []string{ofqn, parsed.ObjName} } -func (lom *LOM) PushFntl(temp []string) (saved []string) { +func (lom *LOM) PushFntl(short []string) (saved []string) { saved = []string{lom.FQN, lom.ObjName} - lom.FQN, lom.ObjName = temp[0], temp[1] + lom.FQN, lom.ObjName = short[0], short[1] lom.HrwFQN = &lom.FQN return saved } diff --git a/fs/content.go b/fs/content.go index f142ad8169e..d4b35bada4b 100644 --- a/fs/content.go +++ b/fs/content.go @@ -41,6 +41,7 @@ const ( ) // file name too long (0x24) +// see FnameTooLong below const ( PrefixFntl = ".x" @@ -122,6 +123,15 @@ func (f *contentSpecMgr) _reg(contentType string, spec ContentResolver) error { return nil } +func FnameTooLong(objName string) bool { + if l := len(objName); l > maxLenBasename { + if l > maxLenPath || len(filepath.Base(objName)) > maxLenBasename { + return true + } + } + return false +} + // Gen returns a new FQN generated from given parts. func (f *contentSpecMgr) Gen(parts PartsFQN, contentType, prefix string) (fqn string) { var ( @@ -133,11 +143,9 @@ func (f *contentSpecMgr) Gen(parts PartsFQN, contentType, prefix string) (fqn st // override caller-provided `objName` to prevent "file name too long" errno 0x24 // - full pathname should be fine as (validated) bucket name <= 64 // - see related: core/lom and "fixup fntl" - if l := len(objName); l > maxLenBasename { - if l > maxLenPath || len(filepath.Base(objName)) > maxLenBasename { - nlog.Warningln("file name too long (0x24):", objName) - objName = PrefixFntl + cos.ChecksumB2S(cos.UnsafeB(objName), cos.ChecksumSHA256) - } + if FnameTooLong(objName) { + nlog.Warningln("file name too long (0x24):", objName) + objName = PrefixFntl + cos.ChecksumB2S(cos.UnsafeB(objName), cos.ChecksumSHA256) } return parts.Mountpath().MakePathFQN(parts.Bucket(), contentType, objName) diff --git a/tools/client.go b/tools/client.go index 3075619b469..1a7109e4e11 100644 --- a/tools/client.go +++ b/tools/client.go @@ -64,6 +64,7 @@ type PutObjectsArgs struct { CksumType string ObjSize uint64 ObjCnt int + ObjNameLn int WorkerCnt int FixedSize bool Ordered bool // true - object names make sequence, false - names are random @@ -334,7 +335,8 @@ func PutRandObjs(args PutObjectsArgs) ([]string, int, error) { if args.Ordered { objNames = append(objNames, path.Join(args.ObjPath, strconv.Itoa(i))) } else { - objNames = append(objNames, path.Join(args.ObjPath, trand.String(16))) + nameLen := cos.NonZero(args.ObjNameLn, 16) + objNames = append(objNames, path.Join(args.ObjPath, trand.String(nameLen))) } } chunkSize := (len(objNames) + workerCnt - 1) / workerCnt