core: remove "file name too long" limitation (ref)

* limit mountpath length * refactor * part six, prev. commit: 4d75c7d Signed-off-by: Alex Aizman <[email protected]>
NVIDIA · Jan 11, 2025 · 455bb66 · 455bb66
1 parent 7fe1d0b
commit 455bb66
Show file tree

Hide file tree

Showing 5 changed files with 62 additions and 32 deletions.
diff --git a/cmn/config.go b/cmn/config.go
@@ -1535,7 +1535,7 @@ func (c *FSPConf) MarshalJSON() ([]byte, error) {
 
 func (c *FSPConf) Validate(contextConfig *Config) error {
 	debug.Assertf(cos.StringInSlice(contextConfig.role, []string{apc.Proxy, apc.Target}),
-		"unexpected role: %q", contextConfig.role)
+		"unexpected node type: %q", contextConfig.role)
 
 	// Don't validate in testing environment.
 	if contextConfig.TestingEnv() || contextConfig.role != apc.Target {
@@ -1637,6 +1637,11 @@ func (c *TestFSPConf) ValidateMpath(p string) (err error) {
 }
 
 // common mountpath validation (NOTE: calls filepath.Clean() every time)
+
+const (
+	maxLenMountpath = 255
+)
+
 func ValidateMpath(mpath string) (string, error) {
 	cleanMpath := filepath.Clean(mpath)
 
@@ -1646,6 +1651,9 @@ func ValidateMpath(mpath string) (string, error) {
 	if cleanMpath == cos.PathSeparator {
 		return "", NewErrInvalidaMountpath(mpath, "root directory is not a valid mountpath")
 	}
+	if len(cleanMpath) > maxLenMountpath {
+		return "", NewErrInvalidaMountpath(mpath, "mountpath length cannot exceed "+strconv.Itoa(maxLenMountpath))
+	}
 	return cleanMpath, nil
 }
 

diff --git a/core/lom.go b/core/lom.go
@@ -675,27 +675,23 @@ func (lom *LOM) IsFntl() bool {
 	return lom.md.lid.haslmfl(lmflFntl)
 }
 
-func (lom *LOM) HasFntlPrefix() bool {
-	return strings.HasPrefix(lom.ObjName, fs.PrefixFntl)
-}
-
 // (compare with fs/content Gen())
 func (lom *LOM) ShortenFntl() []string {
-	debug.Assert(fs.FnameTooLong(lom.ObjName), lom.FQN)
+	debug.Assert(fs.IsFntl(lom.ObjName), lom.FQN)
 
-	noname := fs.PrefixFntl + cos.ChecksumB2S(cos.UnsafeB(lom.FQN), cos.ChecksumSHA256)
+	noname := fs.ShortenFntl(lom.FQN)
 	nfqn := lom.mi.MakePathFQN(lom.Bucket(), fs.ObjectType, noname)
 
 	debug.Assert(len(nfqn) < 4096, "PATH_MAX /usr/include/limits.h", len(nfqn))
 	return []string{nfqn, noname}
 }
 
 func (lom *LOM) fixupFntl() {
-	if !fs.FnameTooLong(lom.ObjName) {
+	if !fs.IsFntl(lom.ObjName) {
 		return
 	}
-	lom.ObjName = fs.PrefixFntl + cos.ChecksumB2S(cos.UnsafeB(lom.FQN), cos.ChecksumSHA256) // noname
-	lom.FQN = lom.mi.MakePathFQN(lom.Bucket(), fs.ObjectType, lom.ObjName)                  // nfqn
+	lom.ObjName = fs.ShortenFntl(lom.FQN)                                  // noname
+	lom.FQN = lom.mi.MakePathFQN(lom.Bucket(), fs.ObjectType, lom.ObjName) // nfqn
 	lom.HrwFQN = &lom.FQN
 }
 

diff --git a/fs/content.go b/fs/content.go
@@ -40,15 +40,6 @@ const (
 	ECMetaType   = "mt"
 )
 
-// file name too long (0x24)
-// see FnameTooLong below
-const (
-	PrefixFntl = ".x"
-
-	maxLenBasename = 255
-	maxLenPath     = 4000 // ref: PATH_MAX = 4096 in /usr/include/limits.h
-)
-
 type (
 	ContentResolver interface {
 		// Generates unique base name for original one. This function may add
@@ -123,15 +114,6 @@ func (f *contentSpecMgr) _reg(contentType string, spec ContentResolver) error {
 	return nil
 }
 
-func FnameTooLong(objName string) bool {
-	if l := len(objName); l > maxLenBasename {
-		if l > maxLenPath || len(filepath.Base(objName)) > maxLenBasename {
-			return true
-		}
-	}
-	return false
-}
-
 // Gen returns a new FQN generated from given parts.
 func (f *contentSpecMgr) Gen(parts PartsFQN, contentType, prefix string) (fqn string) {
 	var (
@@ -143,9 +125,9 @@ func (f *contentSpecMgr) Gen(parts PartsFQN, contentType, prefix string) (fqn st
 	// override caller-provided  `objName` to prevent "file name too long" errno 0x24
 	// - full pathname should be fine as (validated) bucket name <= 64
 	// - see related: core/lom and "fixup fntl"
-	if FnameTooLong(objName) {
-		nlog.Warningln("file name too long (0x24):", objName)
-		objName = PrefixFntl + cos.ChecksumB2S(cos.UnsafeB(objName), cos.ChecksumSHA256)
+	if IsFntl(objName) {
+		nlog.Warningln("fntl:", objName)
+		objName = ShortenFntl(objName)
 	}
 
 	return parts.Mountpath().MakePathFQN(parts.Bucket(), contentType, objName)

diff --git a/fs/fntl.go b/fs/fntl.go
@@ -0,0 +1,44 @@
+// Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content
+/*
+ * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+ */
+package fs
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/aistore/cmn/cos"
+)
+
+// file name too long (0x24)
+const (
+	prefixFntl = ".x"
+
+	// a typical local FS limitation
+	maxLenBasename = 255
+
+	// a.k.a. maximum FQN length before we decide to shorten it
+	// given that Linux PATH_MAX = 4096 (see /usr/include/limits.h)
+	// this number also takes into account:
+	// - maxLenMountpath = 255 (see cmn/config)
+	// - max bucket name = 64  (see cos.CheckAlphaPlus)
+	maxLenPath = 3072
+)
+
+func HasPrefixFntl(s string) bool {
+	return strings.HasPrefix(s, prefixFntl)
+}
+
+func ShortenFntl(s string) string {
+	return prefixFntl + cos.ChecksumB2S(cos.UnsafeB(s), cos.ChecksumSHA256)
+}
+
+func IsFntl(objName string) bool {
+	if l := len(objName); l > maxLenBasename {
+		if l > maxLenPath || len(filepath.Base(objName)) > maxLenBasename {
+			return true
+		}
+	}
+	return false
+}
diff --git a/xact/xs/wi_lso.go b/xact/xs/wi_lso.go
@@ -162,7 +162,7 @@ func (wi *walkInfo) _cb(lom *core.LOM, fqn string) (*cmn.LsoEnt, error) {
 	}
 
 	// shortcut #1: name-only optimizes-out loading md (NOTE: won't show misplaced and copies)
-	if wi.msg.IsFlagSet(apc.LsNameOnly) && !lom.HasFntlPrefix() {
+	if wi.msg.IsFlagSet(apc.LsNameOnly) && !fs.HasPrefixFntl(lom.ObjName) {
 		if !isOK(status) {
 			return nil, nil
 		}