core: remove "file name too long" limitation (part one)

Signed-off-by: Alex Aizman <[email protected]>

ais/tgtimpl.go

@@ -62,6 +62,17 @@ func (t *target) Backend(bck *meta.Bck) core.Backend {
 func (t *target) PutObject(lom *core.LOM, params *core.PutParams) error {
 	debug.Assert(params.WorkTag != "" && !params.Atime.IsZero())
 	workFQN := fs.CSM.Gen(lom, fs.WorkfileType, params.WorkTag)
+
+	// TODO -- FIXME: should it stay "short" in memory?
+	if lom.IsFntl() {
+		var (
+			short = lom.ShortenFntl()
+			saved = lom.PushFntl(short)
+		)
+		lom.SetCustomKey(cmn.OrigFntl, saved[0])
+		workFQN = fs.CSM.Gen(lom, fs.WorkfileType, params.WorkTag)
+	}
+
 	poi := allocPOI()
 	{
 		poi.t = t

cmn/cos/err.go

@@ -1,6 +1,6 @@
 // Package cos provides common low-level types and utilities for all aistore projects
 /*
- * Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
+ * Copyright (c) 2018-2025, NVIDIA CORPORATION. All rights reserved.
  */
 package cos
 
@@ -14,6 +14,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 	ratomic "sync/atomic"
 	"syscall"
@@ -156,10 +157,18 @@ func IsErrSyscallTimeout(err error) bool {
 }
 
 func IsPathErr(err error) (ok bool) {
-	if pathErr := (*iofs.PathError)(nil); errors.As(err, &pathErr) {
+	pathErr := (*iofs.PathError)(nil)
+	if errors.As(err, &pathErr) {
 		ok = true
 	}
-	return
+	return ok
+}
+
+// "file name too long" errno 0x24 (36); either one of the two possible reasons:
+// - len(pathname) > PATH_MAX = 4096
+// - len(basename) > 255
+func IsFntl(err error) bool {
+	return strings.Contains(err.Error(), "too long") && errors.Is(err, syscall.ENAMETOOLONG)
 }
 
 // likely out of socket descriptors

cmn/objattrs.go

@@ -38,6 +38,9 @@ const (
 
 	// additional backend
 	LastModified = "LastModified"
+
+	// as the name implies
+	OrigFntl = "orig_fntl"
 )
 
 // object properties

cmn/ver_const.go

@@ -9,16 +9,16 @@ import "github.com/NVIDIA/aistore/cmn/jsp"
 
 const GitHubHome = "https://github.com/NVIDIA/aistore"
 
-// ========================== IMPORTANT NOTE ==============================
+// ========================== NOTE =====================================
 //
 // - (major.minor) version indicates the current version of AIS software
 //   and is updated manually prior to each release;
 //   making a build with an updated version is the precondition to
 //   creating the corresponding git tag
 //
 // - MetaVer* constants, on the other hand, specify all current on-disk
-//   formatting versions (meta-versions) with the intent to support backward
-//   compatibility in the future
+//   formatting versions (meta-versions) with the intent to support
+//   backward compatibility in the future
 //
 // - Most of the enumerated types below utilize `jsp` package to serialize
 //   and format their (versioned) instances. In its turn, `jsp` itself
@@ -32,16 +32,15 @@ const (
 	VersionAuthN   = "1.1"
 )
 
+// NOTE: for (local) LOM meta-versions, see core/lom*
+
 const (
 	MetaverSmap  = 2 // Smap (cluster map) formatting version a.k.a. meta-version (see core/meta/jsp.go)
 	MetaverBMD   = 2 // BMD (bucket metadata) --/--
 	MetaverRMD   = 1 // Rebalance MD (jsp)
 	MetaverVMD   = 2 // Volume MD (jsp)
 	MetaverEtlMD = 1 // ETL MD (jsp)
 
-	MetaverLOM   = 1 // LOM
-	MetaverChunk = 2 // LOM chunk
-
 	MetaverConfig      = 4 // Global Configuration (jsp)
 	MetaverAuthNConfig = 1 // Authn config (jsp) // ditto
 	MetaverAuthTokens  = 1 // Authn tokens (jsp) // ditto

core/lom.go

@@ -53,7 +53,7 @@ type (
 		uname  *string
 		cmn.ObjAttrs
 		atimefs uint64 // (high bit `lomDirtyMask` | int64: atime)
-		lid     lomBID
+		lid     lomBID // (for bitwise structure, see lombid.go)
 	}
 	LOM struct {
 		mi      *fs.Mountpath
@@ -428,9 +428,10 @@ func (lom *LOM) Load(cacheit, locked bool) error {
 		return err
 	}
 	if lom.bid() == 0 {
-		// copies, etc.
+		// NOTE: always zero - not storing it with MetaverLOM = 1
 		lom.setbid(lom.Bprops().BID)
 	}
+
 	if err := lom._checkBucket(bmd); err != nil {
 		return err
 	}
@@ -475,7 +476,7 @@ func (lom *LOM) LoadUnsafe() (err error) {
 	// either a) handled or b) benign from the caller's perspective
 	if _, err = lom.lmfs(true); err == nil {
 		if lom.bid() == 0 {
-			// copies, etc.
+			// ditto (MetaverLOM = 1)
 			lom.setbid(lom.Bprops().BID)
 		}
 		err = lom._checkBucket(bmd)
@@ -559,16 +560,37 @@ func (lom *LOM) FromFS() error {
 	size, atimefs, _, err := lom.Fstat(true /*get-atime*/)
 	if err != nil {
 		if !os.IsNotExist(err) {
-			if cos.IsPathErr(err) && strings.Contains(err.Error(), "not a directory") {
-				// e.g. err "stat .../aaa/111: not a directory" when there's existing ".../aaa" object
-				err := fmt.Errorf("%w (path error)", err)
-				return err
+			switch {
+			// e.g. err "stat .../aaa/111: not a directory" when there's existing ".../aaa" object
+			case strings.Contains(err.Error(), "not a directory") && cos.IsPathErr(err):
+				return fmt.Errorf("%w (object in the path?)", err)
+			case cos.IsFntl(err):
+				lom.md.lid = lomBID(lom.Bprops().BID)
+				lom.md.lid = lom.md.lid.setlmfl(lmflFntl)
+
+				// temp substitute to check existence
+				short := lom.ShortenFntl()
+				saved := lom.PushFntl(short)
+				size, atimefs, _, err = lom.Fstat(true)
+				lom.PopFntl(saved)
+
+				if err == nil {
+					goto exist
+				}
+				debug.Assert(!cos.IsFntl(err))
+				if os.IsNotExist(err) {
+					return err
+				}
+				lom.md.lid = lom.md.lid.clrlmfl(lmflFntl)
+				fallthrough
+			default:
+				err = os.NewSyscallError("stat", err)
+				T.FSHC(err, lom.Mountpath(), lom.FQN)
 			}
-			err = os.NewSyscallError("stat", err)
-			T.FSHC(err, lom.Mountpath(), lom.FQN)
 		}
 		return err
 	}
+exist:
 	if _, err = lom.lmfs(true); err != nil {
 		// retry once
 		if cmn.IsErrLmetaNotFound(err) {
@@ -638,3 +660,46 @@ func (lom *LOM) Unlock(exclusive bool) {
 	nlc := lom.getLocker()
 	nlc.Unlock(lom.Uname(), exclusive)
 }
+
+//
+// file name too long (0x24) ----------------------------
+//
+
+const (
+	prefixFntl = ".%"
+)
+
+func (lom *LOM) IsFntl() bool { return lom.md.lid.haslmfl(lmflFntl) }
+
+func (lom *LOM) ShortenFntl() []string {
+	noname := prefixFntl + cos.ChecksumB2S(cos.UnsafeB(lom.FQN), cos.ChecksumSHA256)
+	nfqn := lom.mi.MakePathFQN(lom.Bucket(), fs.ObjectType, noname)
+
+	debug.Assert(len(nfqn) < 4096, "PATH_MAX /usr/include/limits.h", len(nfqn))
+	return []string{nfqn, noname}
+}
+
+func (lom *LOM) OrigFntl() []string {
+	debug.Assert(lom.IsFntl())
+	ofqn, ok := lom.GetCustomKey(cmn.OrigFntl)
+	if !ok {
+		debug.Assert(false)
+		return nil
+	}
+	var parsed fs.ParsedFQN
+	if err := parsed.Init(ofqn); err != nil {
+		debug.Assert(false)
+		return nil
+	}
+	return []string{ofqn, parsed.ObjName}
+}
+
+func (lom *LOM) PushFntl(temp []string) (saved []string) {
+	saved = []string{lom.FQN, lom.ObjName}
+	lom.FQN, lom.ObjName = temp[0], temp[1]
+	return saved
+}
+
+func (lom *LOM) PopFntl(saved []string) {
+	lom.FQN, lom.ObjName = saved[0], saved[1]
+}

core/lom_xattr.go

@@ -23,6 +23,11 @@ import (
 	"github.com/OneOfOne/xxhash"
 )
 
+const (
+	MetaverLOM   = 1 // LOM
+	MetaverChunk = 2 // LOM chunk // TODO: niy
+)
+
 // On-disk metadata layout - changing any of this must be done with respect
 // to backward compatibility (and with caution).
 //
@@ -40,7 +45,7 @@ import (
 //   on the version of the layout.
 
 // the one and only currently supported checksum type == xxhash;
-// adding more checksums will likely require a new cmn.MetaverLOM version
+// adding more checksums will likely require a new MetaverLOM version
 const mdCksumTyXXHash = 1
 
 // on-disk xattr names
@@ -307,7 +312,7 @@ func (md *lmeta) unpack(buf []byte) error {
 	if len(buf) < prefLen {
 		return fmt.Errorf("%s: too short (%d)", badLmeta, len(buf))
 	}
-	if buf[0] != cmn.MetaverLOM {
+	if buf[0] != MetaverLOM {
 		return fmt.Errorf("%s: unknown version %d", badLmeta, buf[0])
 	}
 	if buf[1] != mdCksumTyXXHash {
@@ -390,7 +395,11 @@ func (md *lmeta) unpack(buf []byte) error {
 			entries := strings.Split(val, customSepa)
 			custom := make(cos.StrKVs, len(entries)/2)
 			for i := 0; i < len(entries); i += 2 {
-				custom[entries[i]] = entries[i+1]
+				key := entries[i]
+				custom[key] = entries[i+1]
+				if key == cmn.OrigFntl {
+					md.lid = md.lid.setlmfl(lmflFntl)
+				}
 			}
 			md.SetCustomMD(custom)
 		default:
@@ -441,7 +450,7 @@ func (md *lmeta) pack(mdSize int64) (buf []byte) {
 	}
 
 	// checksum, prepend, and return
-	buf[0] = cmn.MetaverLOM
+	buf[0] = MetaverLOM
 	buf[1] = mdCksumTyXXHash
 	mdCksumValue := xxhash.Checksum64S(buf[prefLen:], cos.MLCG32)
 	binary.BigEndian.PutUint64(buf[2:], mdCksumValue)

core/lombid.go

@@ -10,28 +10,40 @@ import (
 	"github.com/NVIDIA/aistore/cmn/debug"
 )
 
-const (
-	AisBID = uint64(1 << 63)
-
-	bitshift = 52
-	flagmask = math.MaxUint64 >> bitshift       // 0xfff
-	flagsBID = (flagmask << bitshift) & ^AisBID // 0x7ff0000000000000
-)
-
-type lomBID uint64
-
+// LOM.md.lid layout ======================================================
+//
 // lomBID is a 64-bit field in the LOM - specifically, `lmeta` structure.
 // As the name implies, lomBID is a union of two values:
 // - bucket ID
 // - bitwise flags that apply only and exclusively to _this_ object.
-// As such, lomBID is persistent (with the rest of `lmeta`) and is used for two purposes:
+// As such, lomBID is persistent (with the rest of `lmeta`) and is used for
+// two purposes:
 // - uniquely associate a given object to its containing bucket;
 // - carry optional flags.
+//
 // lomBID bitwise structure:
 // * high bit is reserved for meta.AisBID
 // * next 11 bits: bit flags
 // * remaining (64 - 12) = bitshift bits contain the bucket's serial number.
 
+const (
+	AisBID = uint64(1 << 63)
+
+	bitshift = 52
+	flagmask = math.MaxUint64 >> bitshift       // 0xfff
+	flagsBID = (flagmask << bitshift) & ^AisBID // 0x7ff0000000000000
+)
+
+type (
+	lomFlags uint16
+	lomBID   uint64
+)
+
+const (
+	lmflFntl = lomFlags(1 << iota)
+	lmflReserved
+)
+
 func NewBID(serial uint64, isAis bool) uint64 {
 	// not adding runtime check given the time reasonably
 	// required to create (1 << 52) buckets
@@ -43,20 +55,22 @@ func NewBID(serial uint64, isAis bool) uint64 {
 	return serial
 }
 
-func (lid lomBID) bid() uint64   { return uint64(lid) & ^flagsBID }
-func (lid lomBID) flags() uint16 { return uint16((uint64(lid) & flagsBID) >> bitshift) }
+func (lid lomBID) bid() uint64     { return uint64(lid) & ^flagsBID }
+func (lid lomBID) flags() lomFlags { return lomFlags((uint64(lid) & flagsBID) >> bitshift) }
 
 func (lid lomBID) setbid(bid uint64) lomBID {
 	debug.Assert(bid&flagsBID == 0, bid)
 	return lomBID((uint64(lid) & flagsBID) | bid)
 }
 
-func (lid lomBID) setflags(fl uint16) lomBID {
-	debug.Assert(fl <= uint16(flagsBID>>bitshift), fl)
+func (lid lomBID) setlmfl(fl lomFlags) lomBID {
+	debug.Assert(fl <= lomFlags(flagsBID>>bitshift), fl)
 	return lomBID(uint64(lid) | (uint64(fl) << bitshift))
 }
 
-func (lid lomBID) clrflags(fl uint16) lomBID {
-	debug.Assert(fl <= uint16(flagsBID>>bitshift))
+func (lid lomBID) haslmfl(fl lomFlags) bool { return uint64(lid)&(uint64(fl)<<bitshift) != 0 }
+
+func (lid lomBID) clrlmfl(fl lomFlags) lomBID {
+	debug.Assert(fl <= lomFlags(flagsBID>>bitshift))
 	return lomBID(uint64(lid) & ^(uint64(fl) << bitshift))
 }