core: remove "file name too long" limitation; add test

* part five, prev. commit: d221976

Signed-off-by: Alex Aizman <[email protected]>

ais/test/common_test.go

@@ -68,6 +68,7 @@ type ioContext struct {
 	originalProxyCount  int
 	num                 int
 	numGetsEachFile     int
+	nameLen             int
 	getErrIsFatal       bool
 	silent              bool
 	fixedSize           bool
@@ -232,6 +233,7 @@ func (m *ioContext) puts(ignoreErrs ...bool) {
 		Bck:       m.bck,
 		ObjPath:   m.prefix,
 		ObjCnt:    m.num,
+		ObjNameLn: m.nameLen,
 		ObjSize:   m.fileSize,
 		FixedSize: m.fixedSize,
 		CksumType: p.Cksum.Type,

ais/test/smoke_test.go

@@ -1,6 +1,6 @@
 // Package integration_test.
 /*
- * Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
  */
 package integration_test
 
@@ -11,17 +11,21 @@ import (
 	"github.com/NVIDIA/aistore/core/meta"
 )
 
-func TestSmoke(t *testing.T) {
-	objSizes := [3]uint64{3 * cos.KiB, 19 * cos.KiB, 77 * cos.KiB}
+func TestSmoke(t *testing.T) { _wrd(t, 0) }
+func TestFntl(t *testing.T)  { _wrd(t, 280) }
+
+func _wrd(t *testing.T, nameLen int) {
+	objSizes := [...]uint64{3 * cos.KiB, 19 * cos.KiB, 77 * cos.KiB}
 
 	runProviderTests(t, func(t *testing.T, bck *meta.Bck) {
 		for _, objSize := range objSizes {
-			name := "size:" + cos.ToSizeIEC(int64(objSize), 0)
-			t.Run(name, func(t *testing.T) {
+			tname := "size:" + cos.ToSizeIEC(int64(objSize), 0)
+			t.Run(tname, func(t *testing.T) {
 				m := ioContext{
 					t:        t,
 					bck:      bck.Clone(),
 					num:      100,
+					nameLen:  nameLen,
 					fileSize: objSize,
 					prefix:   "smoke/obj-",
 				}

cmd/cli/cli/multiobj.go

@@ -232,9 +232,11 @@ func _rmOne(c *cli.Context, shift int) error {
 		return err
 	}
 	if shouldHeadRemote(c, bck) {
-		if _, err := headBucket(bck, false /* don't add */); err != nil {
+		bprops, err := headBucket(bck, false /* don't add */)
+		if err != nil {
 			return err
 		}
+		bck.Props = bprops
 	}
 	// [NOTE]
 	// - passing empty bck _not_ to interpret embedded objName as prefix
@@ -262,7 +264,7 @@ func _rmOne(c *cli.Context, shift int) error {
 			qflprn(listFlag), qflprn(templateFlag), qflprn(rmrfFlag))
 	default: // 3. one obj
 		err := api.DeleteObject(apiBP, bck, oltp.objName)
-		if err == nil && oltp.notFound {
+		if err == nil && bck.IsCloud() && oltp.notFound {
 			// [NOTE]
 			// - certain backends return OK when specified object does not exist (see aws.go)
 			// - compensate here

cmd/cli/cli/parse_uri.go

@@ -266,6 +266,10 @@ func dopOLTP(c *cli.Context, bck cmn.Bck, objNameOrTmpl string) (oltp oltp, err
 	case flagIsSet(c, noRecursFlag):
 		oltp.objName = objNameOrTmpl
 
+	case len(objNameOrTmpl) > 255:
+		// not running lsObjVsPref when "prefix-or-objname" is that long
+		oltp.objName = objNameOrTmpl
+
 	default:
 		// [NOTE] additional list-objects call to disambiguate: differentiate embedded prefix from object name
 		dop, err := lsObjVsPref(bck, objNameOrTmpl)

core/lcopy.go

@@ -239,16 +239,9 @@ func (lom *LOM) Copy(mi *fs.Mountpath, buf []byte) (err error) {
 				}
 			}
 		}
-	} else if cos.IsErrFntl(errExists) {
-		// fixup fntl
-		short := lom.ShortenFntl()
-		saved := lom.PushFntl(short)
-		defer lom.PopFntl(saved)
-		copyFQN = mi.MakePathFQN(lom.Bucket(), fs.ObjectType, lom.ObjName)
-		workFQN = mi.MakePathFQN(lom.Bucket(), fs.WorkfileType, fs.WorkfileCopy+"."+lom.ObjName)
 	}
 
-	// copy
+	// do
 	_, _, err = cos.CopyFile(lom.FQN, workFQN, buf, cos.ChecksumNone) // TODO: checksumming
 	if err != nil {
 		return

core/lfile.go

@@ -8,13 +8,11 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	rdebug "runtime/debug"
 	"syscall"
 
 	"github.com/NVIDIA/aistore/cmn"
 	"github.com/NVIDIA/aistore/cmn/cos"
 	"github.com/NVIDIA/aistore/cmn/debug"
-	"github.com/NVIDIA/aistore/cmn/nlog"
 )
 
 const (
@@ -48,7 +46,7 @@ func (lom *LOM) OpenFile() (fh *os.File, _ error) {
 	return fh, nil
 }
 
-// same as above but return reader
+// same as above but return a reader
 func (lom *LOM) Open() (fh cos.LomReader, err error) {
 	fh, err = os.Open(lom.FQN)
 	switch {
@@ -59,12 +57,10 @@ func (lom *LOM) Open() (fh cos.LomReader, err error) {
 			err = e
 		}
 		return nil, err
+
+	// case cos.IsErrFntl(err)
+
 	default:
-		// DEBUG
-		if cos.IsErrFntl(err) {
-			nlog.Errorln(">>>", err)
-			rdebug.PrintStack()
-		}
 		return nil, err
 	}
 }
@@ -149,6 +145,7 @@ func (*LOM) AppendWork(wfqn string) (fh cos.LomWriter, err error) {
 
 func (lom *LOM) RemoveMain() error {
 	return cos.RemoveFile(lom.FQN)
+	// if err != nil && cos.IsErrFntl(err)
 }
 
 func (lom *LOM) RemoveObj(force ...bool) (err error) {

core/lom.go

@@ -417,6 +417,9 @@ func (lom *LOM) Load(cacheit, locked bool) error {
 	// fast path
 	if lmd != nil {
 		lom.md = *lmd
+		if lom.IsFntl() {
+			lom.fixupFntl()
+		}
 		return lom._checkBucket(bmd)
 	}
 
@@ -427,15 +430,15 @@ func (lom *LOM) Load(cacheit, locked bool) error {
 	if err := lom.FromFS(); err != nil {
 		return err
 	}
-	if lom.bid() == 0 {
-		// NOTE: always zero (MetaverLOM = 1: not storing the BID part of the lom.md.lid)
-		lom.setbid(lom.Bprops().BID)
-	}
+
+	// MetaverLOM = 1: always zero (not storing lom.md.lid)
+	debug.Assert(lom.bid() == 0 || lom.bid() == lom.Bprops().BID, lom.bid())
+	lom.setbid(lom.Bprops().BID)
 
 	if err := lom._checkBucket(bmd); err != nil {
 		return err
 	}
-	if cacheit && lcache != nil {
+	if cacheit {
 		md := lom.md
 		lcache.Store(lom.digest, &md)
 	}
@@ -467,16 +470,18 @@ func (lom *LOM) LoadUnsafe() (err error) {
 	// fast path
 	if lmd != nil {
 		lom.md = *lmd
+		if lom.IsFntl() {
+			lom.fixupFntl()
+		}
 		return lom._checkBucket(bmd)
 	}
 
 	// read and decode xattr; NOTE: fs.GetXattr* vs fs.SetXattr race possible and must be
 	// either a) handled or b) benign from the caller's perspective
 	if _, err = lom.lmfs(true); err == nil {
-		if lom.bid() == 0 {
-			// ditto MetaverLOM = 1
-			lom.setbid(lom.Bprops().BID)
-		}
+		// MetaverLOM = 1: always zero (not storing lom.md.lid)
+		debug.Assert(lom.bid() == 0 || lom.bid() == lom.Bprops().BID, lom.bid())
+		lom.setbid(lom.Bprops().BID)
 		err = lom._checkBucket(bmd)
 	}
 	return err
@@ -674,15 +679,26 @@ func (lom *LOM) HasFntlPrefix() bool {
 	return strings.HasPrefix(lom.ObjName, fs.PrefixFntl)
 }
 
+// (compare with fs/content Gen())
 func (lom *LOM) ShortenFntl() []string {
-	// (compare with fs/content Gen())
+	debug.Assert(fs.FnameTooLong(lom.ObjName), lom.FQN)
+
 	noname := fs.PrefixFntl + cos.ChecksumB2S(cos.UnsafeB(lom.FQN), cos.ChecksumSHA256)
 	nfqn := lom.mi.MakePathFQN(lom.Bucket(), fs.ObjectType, noname)
 
 	debug.Assert(len(nfqn) < 4096, "PATH_MAX /usr/include/limits.h", len(nfqn))
 	return []string{nfqn, noname}
 }
 
+func (lom *LOM) fixupFntl() {
+	if !fs.FnameTooLong(lom.ObjName) {
+		return
+	}
+	lom.ObjName = fs.PrefixFntl + cos.ChecksumB2S(cos.UnsafeB(lom.FQN), cos.ChecksumSHA256) // noname
+	lom.FQN = lom.mi.MakePathFQN(lom.Bucket(), fs.ObjectType, lom.ObjName)                  // nfqn
+	lom.HrwFQN = &lom.FQN
+}
+
 func (lom *LOM) OrigFntl() []string {
 	debug.Assert(lom.IsFntl())
 	ofqn, ok := lom.GetCustomKey(cmn.OrigFntl)
@@ -698,9 +714,9 @@ func (lom *LOM) OrigFntl() []string {
 	return []string{ofqn, parsed.ObjName}
 }
 
-func (lom *LOM) PushFntl(temp []string) (saved []string) {
+func (lom *LOM) PushFntl(short []string) (saved []string) {
 	saved = []string{lom.FQN, lom.ObjName}
-	lom.FQN, lom.ObjName = temp[0], temp[1]
+	lom.FQN, lom.ObjName = short[0], short[1]
 	lom.HrwFQN = &lom.FQN
 	return saved
 }

fs/content.go

@@ -41,6 +41,7 @@ const (
 )
 
 // file name too long (0x24)
+// see FnameTooLong below
 const (
 	PrefixFntl = ".x"
 
@@ -122,6 +123,15 @@ func (f *contentSpecMgr) _reg(contentType string, spec ContentResolver) error {
 	return nil
 }
 
+func FnameTooLong(objName string) bool {
+	if l := len(objName); l > maxLenBasename {
+		if l > maxLenPath || len(filepath.Base(objName)) > maxLenBasename {
+			return true
+		}
+	}
+	return false
+}
+
 // Gen returns a new FQN generated from given parts.
 func (f *contentSpecMgr) Gen(parts PartsFQN, contentType, prefix string) (fqn string) {
 	var (
@@ -133,11 +143,9 @@ func (f *contentSpecMgr) Gen(parts PartsFQN, contentType, prefix string) (fqn st
 	// override caller-provided  `objName` to prevent "file name too long" errno 0x24
 	// - full pathname should be fine as (validated) bucket name <= 64
 	// - see related: core/lom and "fixup fntl"
-	if l := len(objName); l > maxLenBasename {
-		if l > maxLenPath || len(filepath.Base(objName)) > maxLenBasename {
-			nlog.Warningln("file name too long (0x24):", objName)
-			objName = PrefixFntl + cos.ChecksumB2S(cos.UnsafeB(objName), cos.ChecksumSHA256)
-		}
+	if FnameTooLong(objName) {
+		nlog.Warningln("file name too long (0x24):", objName)
+		objName = PrefixFntl + cos.ChecksumB2S(cos.UnsafeB(objName), cos.ChecksumSHA256)
 	}
 
 	return parts.Mountpath().MakePathFQN(parts.Bucket(), contentType, objName)

tools/client.go

@@ -64,6 +64,7 @@ type PutObjectsArgs struct {
 	CksumType string
 	ObjSize   uint64
 	ObjCnt    int
+	ObjNameLn int
 	WorkerCnt int
 	FixedSize bool
 	Ordered   bool // true - object names make sequence, false - names are random
@@ -334,7 +335,8 @@ func PutRandObjs(args PutObjectsArgs) ([]string, int, error) {
 		if args.Ordered {
 			objNames = append(objNames, path.Join(args.ObjPath, strconv.Itoa(i)))
 		} else {
-			objNames = append(objNames, path.Join(args.ObjPath, trand.String(16)))
+			nameLen := cos.NonZero(args.ObjNameLn, 16)
+			objNames = append(objNames, path.Join(args.ObjPath, trand.String(nameLen)))
 		}
 	}
 	chunkSize := (len(objNames) + workerCnt - 1) / workerCnt