From e8df59997934036cd02592116961033ad8003b9f Mon Sep 17 00:00:00 2001 From: fatelei Date: Fri, 3 Feb 2023 16:50:12 +0800 Subject: [PATCH 01/23] fix: fix read diff1 directory Signed-off-by: fatelei --- pkg/fs/layer_node.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/fs/layer_node.go b/pkg/fs/layer_node.go index 970baba..9dabfaa 100644 --- a/pkg/fs/layer_node.go +++ b/pkg/fs/layer_node.go @@ -85,6 +85,7 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) l, err := n.fs.layManager.ResolverMetaLayer(ctx, n.refNode.ref, n.refNode.rawRef, n.digest) if err != nil { + log.G(ctx).Warnf("resolve meta layer failed: %+v", err) return nil, syscall.EIO } @@ -100,10 +101,11 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) }) } + sAttr := defaultDirAttr(&out.Attr) child := &diffNode{ fs: n.fs, } - sAttr := defaultDirAttr(&out.Attr) + copyAttr(&child.attr, &out.Attr) return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { out.Attr.Ino = uint64(ino) child.attr.Ino = uint64(ino) From 2b070b1f90e5b703dfb9180410bc04a93f6080c6 Mon Sep 17 00:00:00 2001 From: fatelei Date: Fri, 3 Mar 2023 15:36:28 +0800 Subject: [PATCH 02/23] fix: fix when mount nydus layer failed not return error Signed-off-by: fatelei --- pkg/fs/layer_node.go | 5 ++--- pkg/manager/manager.go | 23 ++++++++++++++++++----- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/pkg/fs/layer_node.go b/pkg/fs/layer_node.go index 9dabfaa..105c326 100644 --- a/pkg/fs/layer_node.go +++ b/pkg/fs/layer_node.go @@ -86,12 +86,11 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) l, err := n.fs.layManager.ResolverMetaLayer(ctx, n.refNode.ref, n.refNode.rawRef, n.digest) if err != nil { log.G(ctx).Warnf("resolve meta layer failed: %+v", err) - return nil, syscall.EIO } if name == blobLink { - sAttr := layerToAttr(l, &out.Attr) - cn := &blobNode{l: l} + sAttr := layerToAttr(&l.Descriptor, &out.Attr) + cn := &blobNode{l: &l.Descriptor} copyAttr(&cn.attr, &out.Attr) return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { out.Attr.Ino = uint64(ino) diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 78b1822..2223133 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -29,6 +29,13 @@ type nydusMessage struct { Err error } +type MountLayer struct { + Descriptor ocispec.Descriptor + IsMetaLayer bool +} + +var MountMetaLayerFailed = errors.New("mount meta layer failed") + func NewLayerManager(ctx context.Context, rootDir string, hosts source.RegistryHosts, cfg *config.Config) (*LayerManager, error) { verifier, err := signature.NewVerifier(cfg.PublicKeyFile, cfg.ValidateSignature) if err != nil { @@ -110,7 +117,7 @@ func (r *LayerManager) GetLayerInfo(ctx context.Context, refspec reference.Spec, return genLayerInfo(dgst, manifest, imageConfig) } -func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference.Spec, snapshotID string, digest digest.Digest) (*ocispec.Descriptor, error) { +func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference.Spec, snapshotID string, digest digest.Digest) (*MountLayer, error) { // get manifest from cache. manifest, _, err := r.refPool.loadRef(ctx, refspec) if err != nil { @@ -130,14 +137,20 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. return nil, fmt.Errorf("unknown digest %v for ref %q", target, refspec.String()) } + layer := MountLayer{ + Descriptor: target, + IsMetaLayer: false, + } + // Download nydus bootstrap layer and mount it. if _, ok := target.Annotations[label.NydusMetaLayer]; ok { target.Annotations[label.CRIImageRef] = refspec.String() target.Annotations[label.CRILayerDigest] = target.Digest.String() + layer.IsMetaLayer = true if _, ok = r.nydusMetaLayer.Load(snapshotID); ok { log.G(ctx).Warnf("nydus duplicate mount meta layer ref is %s digest is %s", refspec.String(), target.Digest.String()) - return &target, nil + return &layer, nil } workdir := r.nydusFs.UpperPath(snapshotID) @@ -180,7 +193,7 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. err = r.nydusFs.WaitUntilReady(ctx, snapshotID) if err != nil { - return nil, err + return nil, MountMetaLayerFailed } // Link nydusd mount dir to /// @@ -190,7 +203,7 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. cmd := exec.Command("mount", "-o", "bind,ro", mountPoint, targetPath) if err = cmd.Start(); err == nil { r.nydusMetaLayer.Store(snapshotID, targetPath) - return &target, nil + return &layer, nil } log.G(ctx).WithError(err).Error("mount bind file has error") return nil, err @@ -199,7 +212,7 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. return nil, err } // TODO support normal image format. - return &target, nil + return &layer, nil } func (r *LayerManager) Release(ctx context.Context, refspec reference.Spec, dgst digest.Digest, snapshotID string) (int, error) { From cef0ca74efcdf1855ec5eb2633d65a52e3a0afde Mon Sep 17 00:00:00 2001 From: fatelei Date: Fri, 3 Mar 2023 15:43:35 +0800 Subject: [PATCH 03/23] fix: fix nil Signed-off-by: fatelei --- pkg/manager/manager.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 2223133..48ef3dd 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -32,6 +32,7 @@ type nydusMessage struct { type MountLayer struct { Descriptor ocispec.Descriptor IsMetaLayer bool + MountFailed bool } var MountMetaLayerFailed = errors.New("mount meta layer failed") @@ -157,7 +158,7 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. if _, err = os.Stat(workdir); os.IsNotExist(err) { if err = os.MkdirAll(workdir, 0755); err != nil { log.G(ctx).WithError(err).Error("mkdir nydus snapshot dir failed") - return nil, err + return &layer, err } } @@ -165,7 +166,7 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. err = r.nydusFs.PrepareMetaLayer(ctx, storage.Snapshot{ID: snapshotID}, target.Annotations) if err != nil && !strings.Contains(err.Error(), "file exists") { log.G(ctx).WithError(err).Error("download snapshot files failed") - return nil, err + return &layer, err } nydusMsgChannel := make(chan nydusMessage) @@ -188,12 +189,12 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. event := <-nydusMsgChannel if event.Err != nil { - return nil, event.Err + return &layer, event.Err } err = r.nydusFs.WaitUntilReady(ctx, snapshotID) if err != nil { - return nil, MountMetaLayerFailed + return &layer, MountMetaLayerFailed } // Link nydusd mount dir to /// @@ -206,10 +207,10 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. return &layer, nil } log.G(ctx).WithError(err).Error("mount bind file has error") - return nil, err + return &layer, err } log.G(ctx).WithError(err).Error("get mount point failed") - return nil, err + return &layer, err } // TODO support normal image format. return &layer, nil From 638e178fa4c18667f78f9ad2c38705881ba43719 Mon Sep 17 00:00:00 2001 From: fatelei Date: Sun, 26 Oct 2025 21:12:38 +0800 Subject: [PATCH 04/23] refactor: refactor this plugin Signed-off-by: fatelei --- cmd/store/main.go | 10 ++-------- pkg/fs/layer_node.go | 14 ++++++++++++-- pkg/manager/manager.go | 44 +++++++++++++++++++++++------------------- 3 files changed, 38 insertions(+), 30 deletions(-) diff --git a/cmd/store/main.go b/cmd/store/main.go index d68cf2f..927be87 100644 --- a/cmd/store/main.go +++ b/cmd/store/main.go @@ -44,14 +44,8 @@ func main() { } mountPoint := fmt.Sprintf("%s/store", flags.Args.RootDir) - if _, err := os.Stat(mountPoint); err != nil { - if os.IsNotExist(err) { - if err := os.MkdirAll(mountPoint, 0755); err != nil { - return errors.Wrapf(err, "create root directory %s", mountPoint) - } - } else { - return errors.Wrapf(err, "stat root directory %s", mountPoint) - } + if err := os.MkdirAll(mountPoint, 0755); err != nil { + return errors.Wrapf(err, "create root directory %s", mountPoint) } // replace it with nydus-snapshotter resolver. diff --git a/pkg/fs/layer_node.go b/pkg/fs/layer_node.go index 105c326..7d1abc3 100644 --- a/pkg/fs/layer_node.go +++ b/pkg/fs/layer_node.go @@ -24,11 +24,10 @@ type layerNode struct { digest digest.Digest } -var _ = (fusefs.NodeGetattrer)((*diffNode)(nil)) -var _ = (fusefs.InodeEmbedder)((*diffNode)(nil)) var _ = (fusefs.InodeEmbedder)((*layerNode)(nil)) var _ = (fusefs.NodeCreater)((*layerNode)(nil)) var _ = (fusefs.NodeLookuper)((*layerNode)(nil)) +var _ = (fusefs.NodeReaddirer)((*layerNode)(nil)) // Create marks this layer as "using". // We don't use refnode.Mkdir because Mkdir event doesn't reach here if layernode already exists. @@ -131,3 +130,14 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) return nil, syscall.ENOENT } } + +// Readdir enumerates expected entries to help consumers like Podman discover files reliably. +func (n *layerNode) Readdir(ctx context.Context) (fusefs.DirStream, syscall.Errno) { + entries := []fuse.DirEntry{ + {Name: layerInfoLink, Mode: fuse.S_IFREG}, + {Name: blobLink, Mode: fuse.S_IFREG}, + {Name: layerLink, Mode: fuse.S_IFDIR}, + {Name: layerUseFile, Mode: fuse.S_IFREG}, + } + return fusefs.NewListDirStream(entries), 0 +} diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 48ef3dd..4b54e58 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "os/exec" "path/filepath" "strings" "sync" @@ -21,6 +20,7 @@ import ( "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "golang.org/x/sys/unix" "github.com/containers/nydus-storage-plugin/pkg/source" ) @@ -85,9 +85,6 @@ func NewLayerManager(ctx context.Context, rootDir string, hosts source.RegistryH if err != nil { return nil, err } - if err != nil { - return nil, fmt.Errorf("failed to setup resolver: %w", err) - } return &LayerManager{ refPool: refPool, hosts: hosts, @@ -197,20 +194,30 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. return &layer, MountMetaLayerFailed } - // Link nydusd mount dir to /// - targetPath := fmt.Sprintf("%s/store/%s/%s/diff", r.rootDir, snapshotID, target.Digest.String()) - var mountPoint string - if mountPoint, err = r.nydusFs.MountPoint(snapshotID); err == nil { - cmd := exec.Command("mount", "-o", "bind,ro", mountPoint, targetPath) - if err = cmd.Start(); err == nil { + // Link nydusd mount dir to ///diff + targetPath := filepath.Join(r.rootDir, "store", snapshotID, target.Digest.String(), "diff") + var mountPoint string + if mountPoint, err = r.nydusFs.MountPoint(snapshotID); err == nil { + if err := os.MkdirAll(targetPath, 0755); err != nil { + log.G(ctx).WithError(err).Error("ensure targetPath failed") + return &layer, err + } + // Perform a bind mount and then remount read-only for robustness across kernels + if err = unix.Mount(mountPoint, targetPath, "", unix.MS_BIND|unix.MS_REC, ""); err != nil { + log.G(ctx).WithError(err).Error("bind mount failed") + return &layer, err + } + if err = unix.Mount("", targetPath, "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY, ""); err != nil { + log.G(ctx).WithError(err).Error("remount ro failed") + // try to unmount in case remount failed partially + _ = unix.Unmount(targetPath, 0) + return &layer, err + } r.nydusMetaLayer.Store(snapshotID, targetPath) return &layer, nil } - log.G(ctx).WithError(err).Error("mount bind file has error") + log.G(ctx).WithError(err).Error("get mount point failed") return &layer, err - } - log.G(ctx).WithError(err).Error("get mount point failed") - return &layer, err } // TODO support normal image format. return &layer, nil @@ -233,16 +240,14 @@ func (r *LayerManager) Release(ctx context.Context, refspec reference.Spec, dgst i := r.refCounter[refspec.String()][dgst.String()] if i <= 0 { if v, ok := r.nydusMetaLayer.Load(snapshotID); ok { - cmd := exec.Command("umount", v.(string)) - if err := cmd.Run(); err != nil { + if err := unix.Unmount(v.(string), 0); err != nil { log.G(ctx).Errorf("umount bind nydus %v/%v failed: %+v", refspec, dgst, err) return 0, err } r.nydusMetaLayer.Delete(snapshotID) } - // No reference to this layer. release it. - delete(r.refCounter, dgst.String()) + delete(r.refCounter[refspec.String()], dgst.String()) if len(r.refCounter[refspec.String()]) == 0 { delete(r.refCounter, refspec.String()) } @@ -253,8 +258,7 @@ func (r *LayerManager) Release(ctx context.Context, refspec reference.Spec, dgst func (r *LayerManager) ReleaseAll(ctx context.Context) { r.nydusMetaLayer.Range(func(key, value interface{}) bool { - cmd := exec.Command("umount", value.(string)) - if err := cmd.Run(); err != nil { + if err := unix.Unmount(value.(string), 0); err != nil { log.G(ctx).WithError(err).Warnf("umount bind nydus %v/%v failed", key, value) } return true From c0419d2df60b47d8b506cfe1bfa4acbfa7b4af4d Mon Sep 17 00:00:00 2001 From: fatelei Date: Sun, 26 Oct 2025 22:53:22 +0800 Subject: [PATCH 05/23] feat: add ut and bump go to 1.25 and FUSE3: auto-detects fusermount or fusermount3; falls back to direct mount if neither found. Configurable modes: new flags --fs-file-mode, --fs-dir-mode, --fs-link-mode (octal, e.g. 0400/0500). Non-nydus clarity: returns ENOENT for diff on non-nydus layers; blob/info still available. Crash recovery: best-effort unmount of orphan bind mounts at startup. Signed-off-by: fatelei --- cmd/store/main.go | 85 ++++++-- go.mod | 2 +- pkg/fs/blob_file_test.go | 23 +++ pkg/fs/blob_node_test.go | 17 ++ pkg/fs/diff_node_test.go | 27 +++ pkg/fs/fs.go | 63 ++++-- pkg/fs/fs_test.go | 3 + pkg/fs/id_map_test.go | 37 ++++ pkg/fs/layer_node.go | 26 ++- pkg/fs/layer_node_more_test.go | 18 ++ pkg/fs/layer_node_test.go | 41 ++++ pkg/fs/ref_node.go | 16 +- pkg/fs/ref_node_test.go | 13 ++ pkg/fs/root_node.go | 10 +- pkg/fs/root_node_test.go | 3 + pkg/fs/utils_test.go | 48 +++++ pkg/manager/manager.go | 85 +++++--- pkg/manager/manager_test.go | 185 ++++++++++++++++++ pkg/manager/mount_shim_linux.go | 24 +++ pkg/manager/mount_shim_other.go | 19 ++ pkg/manager/ref_pool.go | 12 +- .../keychain/dockerconfig/dockerconfig.go | 4 +- .../keychain/podmanauth/podmanauth.go | 110 +++++++++++ 23 files changed, 792 insertions(+), 79 deletions(-) create mode 100644 pkg/fs/blob_file_test.go create mode 100644 pkg/fs/blob_node_test.go create mode 100644 pkg/fs/diff_node_test.go create mode 100644 pkg/fs/fs_test.go create mode 100644 pkg/fs/id_map_test.go create mode 100644 pkg/fs/layer_node_more_test.go create mode 100644 pkg/fs/layer_node_test.go create mode 100644 pkg/fs/ref_node_test.go create mode 100644 pkg/fs/root_node_test.go create mode 100644 pkg/fs/utils_test.go create mode 100644 pkg/manager/manager_test.go create mode 100644 pkg/manager/mount_shim_linux.go create mode 100644 pkg/manager/mount_shim_other.go create mode 100644 pkg/services/keychain/podmanauth/podmanauth.go diff --git a/cmd/store/main.go b/cmd/store/main.go index 927be87..fe4bd1f 100644 --- a/cmd/store/main.go +++ b/cmd/store/main.go @@ -5,10 +5,10 @@ import ( "os" "os/signal" "syscall" + "log/slog" + "path/filepath" - "github.com/containerd/containerd/log" "github.com/containerd/nydus-snapshotter/cmd/containerd-nydus-grpc/pkg/command" - "github.com/containerd/nydus-snapshotter/cmd/containerd-nydus-grpc/pkg/logging" "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/pkg/errdefs" "github.com/pkg/errors" @@ -17,8 +17,9 @@ import ( "github.com/containers/nydus-storage-plugin/pkg/fs" "github.com/containers/nydus-storage-plugin/pkg/manager" "github.com/containers/nydus-storage-plugin/pkg/services/keychain/dockerconfig" + podmanauth "github.com/containers/nydus-storage-plugin/pkg/services/keychain/podmanauth" "github.com/containers/nydus-storage-plugin/pkg/services/resolver" -) + ) func waitForSIGINT() { c := make(chan os.Signal, 1) @@ -26,15 +27,51 @@ func waitForSIGINT() { <-c } +func parseOctalMode(s string) (uint32, error) { + var v uint32 + for i := 0; i < len(s); i++ { + c := s[i] + if c < '0' || c > '7' { return 0, fmt.Errorf("invalid octal: %s", s) } + v = (v << 3) | uint32(c-'0') + } + return v, nil +} + +func setupSlog(level string, toStdout bool, logDir string, root string) error { + var lvl slog.Level + switch level { + case "debug": lvl = slog.LevelDebug + case "warn", "warning": lvl = slog.LevelWarn + case "error": lvl = slog.LevelError + default: lvl = slog.LevelInfo + } + var w *os.File + if toStdout || logDir == "" { + w = os.Stdout + } else { + path := filepath.Join(logDir, "nydus-store.log") + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { return err } + w = f + } + h := slog.NewTextHandler(w, &slog.HandlerOptions{Level: lvl}) + slog.SetDefault(slog.New(h)) + return nil +} + func main() { flags := command.NewFlags() app := &cli.App{ Name: "crio nydus store", Usage: "crio nydus store plugin", Version: "0.0.0", - Flags: flags.F, + Flags: append(flags.F, + &cli.StringFlag{Name: "fs-file-mode", Usage: "octal file mode for files (e.g. 0400)"}, + &cli.StringFlag{Name: "fs-dir-mode", Usage: "octal dir mode (e.g. 0500)"}, + &cli.StringFlag{Name: "fs-link-mode", Usage: "octal symlink mode (e.g. 0400)"}, + ), Action: func(c *cli.Context) error { - if err := logging.SetUp(flags.Args.LogLevel, flags.Args.LogToStdout, flags.Args.LogDir, flags.Args.RootDir); err != nil { + if err := setupSlog(flags.Args.LogLevel, flags.Args.LogToStdout, flags.Args.LogDir, flags.Args.RootDir); err != nil { return errors.Wrap(err, "failed to prepare logger") } @@ -49,22 +86,46 @@ func main() { } // replace it with nydus-snapshotter resolver. - hosts := resolver.RegistryHostsFromConfig([]resolver.Credential{dockerconfig.NewDockerconfigKeychain(c.Context)}...) + hosts := resolver.RegistryHostsFromConfig( + []resolver.Credential{ + dockerconfig.NewDockerconfigKeychain(c.Context), + // Podman-compatible auth.json + podmanauth.NewPodmanAuthKeychain(c.Context), + }..., + ) layManager, err := manager.NewLayerManager(c.Context, flags.Args.RootDir, hosts, &cfg) if err != nil { panic(err) } - if err := fs.Mount(c.Context, mountPoint, flags.Args.RootDir, true, layManager); err != nil { - log.G(c.Context).WithError(err).Fatalf("failed to mount fs at %q", mountPoint) + // Parse optional FS modes + fileMode := fs.DefaultFileMode() + dirMode := fs.DefaultDirMode() + linkMode := fs.DefaultLinkMode() + if v := c.String("fs-file-mode"); v != "" { + if m, err := parseOctalMode(v); err == nil { fileMode = m } + } + if v := c.String("fs-dir-mode"); v != "" { + if m, err := parseOctalMode(v); err == nil { dirMode = m } + } + if v := c.String("fs-link-mode"); v != "" { + if m, err := parseOctalMode(v); err == nil { linkMode = m } + } + + // Recover orphan bind mounts from previous crashes + _ = layManager.RecoverOrphanMounts(c.Context) + + if err := fs.Mount(c.Context, mountPoint, flags.Args.RootDir, true, layManager, fs.WithModes(fileMode, dirMode, linkMode)); err != nil { + slog.ErrorContext(c.Context, "failed to mount fs", "mountPoint", mountPoint, "err", err) + return err } defer func() { layManager.ReleaseAll(c.Context) err := syscall.Unmount(mountPoint, 0) if err != nil { - log.G(c.Context).Error(err) + slog.ErrorContext(c.Context, "unmount failed", "err", err) } - log.G(c.Context).Info("Exiting") + slog.InfoContext(c.Context, "Exiting") }() waitForSIGINT() return nil @@ -72,9 +133,9 @@ func main() { } if err := app.Run(os.Args); err != nil { if errdefs.IsConnectionClosed(err) { - log.L.Info("snapshotter exited") + slog.Info("snapshotter exited") return } - log.L.WithError(err).Fatal("failed to start crio nydus store") + slog.Error("failed to start crio nydus store", "err", err) } } diff --git a/go.mod b/go.mod index d82f6ac..07a30cf 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/containers/nydus-storage-plugin -go 1.18 +go 1.25 require ( github.com/containerd/containerd v1.6.19 diff --git a/pkg/fs/blob_file_test.go b/pkg/fs/blob_file_test.go new file mode 100644 index 0000000..cf82104 --- /dev/null +++ b/pkg/fs/blob_file_test.go @@ -0,0 +1,23 @@ +package fs + +import ( + "context" + "testing" + + "github.com/hanwen/go-fuse/v2/fuse" +) + +func TestBlobFileReadEIO(t *testing.T) { + bf := &blobFile{} +if rr, eno := bf.Read(context.Background(), nil, 0); eno == 0 || rr != nil { + t.Fatalf("expected EIO and nil read result, got eno=%d rr=%v", eno, rr) + } +} + +func TestBlobFileGetattrNoError(t *testing.T) { + bf := &blobFile{} + var out fuse.AttrOut +if eno := bf.Getattr(context.Background(), &out); eno != 0 { + t.Fatalf("Getattr returned errno=%d", eno) + } +} diff --git a/pkg/fs/blob_node_test.go b/pkg/fs/blob_node_test.go new file mode 100644 index 0000000..f5509f9 --- /dev/null +++ b/pkg/fs/blob_node_test.go @@ -0,0 +1,17 @@ +package fs + +import ( + "context" + "testing" +) + +func TestBlobNodeOpenReturnsBlobFile(t *testing.T) { + n := &blobNode{} +fh, _, eno := n.Open(context.Background(), 0) + if eno != 0 { + t.Fatalf("Open errno=%d", eno) + } + if _, ok := fh.(*blobFile); !ok { + t.Fatalf("expected *blobFile, got %T", fh) + } +} diff --git a/pkg/fs/diff_node_test.go b/pkg/fs/diff_node_test.go new file mode 100644 index 0000000..3f126f6 --- /dev/null +++ b/pkg/fs/diff_node_test.go @@ -0,0 +1,27 @@ +package fs + +import ( + "context" + "testing" + + "github.com/hanwen/go-fuse/v2/fuse" +) + +func TestDiffNodeGetattrCopiesAttr(t *testing.T) { + d := &diffNode{} + d.attr.Mode = 0o755 + var out fuse.AttrOut + if eno := d.Getattr(context.Background(), nil, &out); eno != 0 { + t.Fatalf("Getattr errno=%d", eno) + } + if out.Attr.Mode != d.attr.Mode { + t.Fatalf("mode mismatch: got %o want %o", out.Attr.Mode, d.attr.Mode) + } +} + +func TestDiffNodeRmdirAlwaysENOENT(t *testing.T) { + d := &diffNode{} + if eno := d.Rmdir(context.Background(), "anything"); eno == 0 { + t.Fatalf("expected ENOENT, got 0") + } +} diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 7675fbd..62cf7d4 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -9,28 +9,49 @@ import ( "syscall" "time" - "github.com/containerd/containerd/log" + "log/slog" + "github.com/containers/nydus-storage-plugin/pkg/manager" fusefs "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) const ( - defaultLinkMode = syscall.S_IFLNK | 0400 // -r-------- - defaultDirMode = syscall.S_IFDIR | 0500 // dr-x------ - defaultFileMode = 0400 // -r-------- - layerFileMode = 0400 // -r-------- - blockSize = 4096 + blockSize = 4096 poolLink = "pool" layerLink = "diff" blobLink = "blob" layerInfoLink = "info" layerUseFile = "use" +) - fusermountBin = "fusermount" +// Configurable FS modes (defaults preserved) +var ( + defaultLinkMode uint32 = syscall.S_IFLNK | 0400 // -r-------- + defaultDirMode uint32 = syscall.S_IFDIR | 0500 // dr-x------ + defaultFileMode uint32 = 0400 // -r-------- + layerFileMode uint32 = 0400 // -r-------- ) +// Helpers to expose defaults for CLI parsing +func DefaultFileMode() uint32 { return defaultFileMode } +func DefaultDirMode() uint32 { return defaultDirMode } +func DefaultLinkMode() uint32 { return defaultLinkMode } + +// Mount options +type MountOption func() + +// WithModes overrides default FS modes at mount time. +func WithModes(fileMode, dirMode, linkMode uint32) MountOption { + return func() { + defaultFileMode = fileMode + defaultDirMode = dirMode + defaultLinkMode = linkMode + layerFileMode = fileMode + } +} + type releasable interface { releasable() bool } @@ -78,7 +99,14 @@ func (r *inoReleasable) releasable() bool { return r.n.EmbeddedInode().Forgotten() } -func Mount(ctx context.Context, mountPoint string, rootDir string, debug bool, layManager *manager.LayerManager) error { +func Mount(ctx context.Context, mountPoint string, rootDir string, debug bool, layManager *manager.LayerManager, opts ...MountOption) error { + // Apply mount options + for _, o := range opts { + if o != nil { + o() + } + } + seconds := time.Second rawFS := fusefs.NewNodeFS(&rootNode{ fs: &fs{ @@ -96,10 +124,11 @@ func Mount(ctx context.Context, mountPoint string, rootDir string, debug bool, l FsName: "nydusstore", Debug: debug, } - if _, err := exec.LookPath(fusermountBin); err == nil { - mountOpts.Options = []string{"suid"} // option for fusermount; allow setuid inside container + // Detect fusermount or fusermount3; fallback to direct mount if neither present + if hasFusermount() { + mountOpts.Options = []string{"suid"} // allow setuid inside container } else { - log.L.WithError(err).Debugf("%s not installed; trying direct mount", fusermountBin) + slog.Debug("fusermount/fusermount3 not installed; trying direct mount") mountOpts.DirectMount = true } server, err := fuse.NewServer(rawFS, mountPoint, mountOpts) @@ -110,13 +139,23 @@ func Mount(ctx context.Context, mountPoint string, rootDir string, debug bool, l return server.WaitMount() } +func hasFusermount() bool { + if _, err := exec.LookPath("fusermount"); err == nil { + return true + } + if _, err := exec.LookPath("fusermount3"); err == nil { + return true + } + return false +} + func (fs *fs) newInodeWithID(ctx context.Context, p func(uint32) fusefs.InodeEmbedder) (*fusefs.Inode, syscall.Errno) { var ino fusefs.InodeEmbedder if err := fs.nodeMap.add(func(id uint32) (releasable, error) { ino = p(id) return &inoReleasable{ino}, nil }); err != nil || ino == nil { - log.L.WithContext(ctx).WithError(err).Debug("cannot generate ID") + slog.DebugContext(ctx, "cannot generate ID", "err", err) return nil, syscall.EIO } return ino.EmbeddedInode(), 0 diff --git a/pkg/fs/fs_test.go b/pkg/fs/fs_test.go new file mode 100644 index 0000000..634f516 --- /dev/null +++ b/pkg/fs/fs_test.go @@ -0,0 +1,3 @@ +package fs + +// newInodeWithID depends on a live go-fuse server; skip unit testing here. diff --git a/pkg/fs/id_map_test.go b/pkg/fs/id_map_test.go new file mode 100644 index 0000000..c7a837a --- /dev/null +++ b/pkg/fs/id_map_test.go @@ -0,0 +1,37 @@ +package fs + +import ( + "testing" +) + +type tr struct{ releasableFlag bool } + +func (r *tr) releasable() bool { return r.releasableFlag } + +func TestIDMapAddAssignsAndReuses(t *testing.T) { + m := &idMap{} + var saved []*tr + add := func(id uint32) (releasable, error) { + r := &tr{} + saved = append(saved, r) + return r, nil + } + if err := m.add(add); err != nil { + t.Fatalf("add #1: %v", err) + } + if err := m.add(add); err != nil { + t.Fatalf("add #2: %v", err) + } + // IDs should start from 2 (skipping 0 and 1), so expect two entries + if len(m.m) != 2 { + t.Fatalf("expected 2 entries, got %d", len(m.m)) + } + // Mark first releasable and add again, expecting reuse of freed ID + saved[0].releasableFlag = true + if err := m.add(add); err != nil { + t.Fatalf("add #3: %v", err) + } + if len(m.m) != 2 { + t.Fatalf("expected still 2 entries after reuse, got %d", len(m.m)) + } +} diff --git a/pkg/fs/layer_node.go b/pkg/fs/layer_node.go index 7d1abc3..c6cfa32 100644 --- a/pkg/fs/layer_node.go +++ b/pkg/fs/layer_node.go @@ -7,8 +7,8 @@ import ( "context" "encoding/json" "syscall" + "log/slog" - "github.com/containerd/containerd/log" fusefs "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" "github.com/opencontainers/go-digest" @@ -34,24 +34,24 @@ var _ = (fusefs.NodeReaddirer)((*layerNode)(nil)) func (n *layerNode) Create(ctx context.Context, name string, flags uint32, mode uint32, out *fuse.EntryOut) (node *fusefs.Inode, fh fusefs.FileHandle, fuseFlags uint32, errno syscall.Errno) { if name == layerUseFile { current := n.fs.layManager.Use(n.refNode.ref, n.digest) - log.G(ctx).WithField("refcounter", current).Infof("layer %v / %v is marked as USING", n.refNode.ref, n.digest) + slog.InfoContext(ctx, "layer marked USING", "ref", n.refNode.ref.String(), "digest", n.digest.String(), "refcounter", current) } return nil, nil, 0, syscall.ENOENT } // Lookup routes to the target file stored in the pool, based on the specified file name. func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fusefs.Inode, syscall.Errno) { - log.L.WithContext(ctx).Debugf("layer node lookup name = %s", name) + slog.DebugContext(ctx, "layer node lookup", "name", name) switch name { case layerInfoLink: info, err := n.fs.layManager.GetLayerInfo(ctx, n.refNode.ref, n.digest) if err != nil { - log.G(ctx).WithError(err).Warnf("failed to get layer info for %q: %q", name, n.digest) + slog.WarnContext(ctx, "failed to get layer info", "name", name, "digest", n.digest.String(), "err", err) return nil, syscall.EIO } buf := new(bytes.Buffer) if err := json.NewEncoder(buf).Encode(&info); err != nil { - log.G(ctx).WithError(err).Warnf("failed to encode layer info for %q: %q", name, n.digest) + slog.WarnContext(ctx, "failed to encode layer info", "name", name, "digest", n.digest.String(), "err", err) return nil, syscall.EIO } infoData := buf.Bytes() @@ -84,9 +84,11 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) l, err := n.fs.layManager.ResolverMetaLayer(ctx, n.refNode.ref, n.refNode.rawRef, n.digest) if err != nil { - log.G(ctx).Warnf("resolve meta layer failed: %+v", err) + slog.WarnContext(ctx, "resolve meta layer failed", "err", err) + if name == layerLink { + return nil, syscall.ENOENT + } } - if name == blobLink { sAttr := layerToAttr(&l.Descriptor, &out.Attr) cn := &blobNode{l: &l.Descriptor} @@ -99,6 +101,12 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) }) } + // Only Nydus meta layers expose a diff directory via bind mount + if !l.IsMetaLayer || l.MountFailed { + slog.DebugContext(ctx, "not a nydus meta layer; no diff provided", "digest", n.digest.String()) + return nil, syscall.ENOENT + } + sAttr := defaultDirAttr(&out.Attr) child := &diffNode{ fs: n.fs, @@ -123,10 +131,10 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) return cn }) case layerUseFile: - log.G(ctx).Debugf("\"use\" file is referred but return ENOENT for reference management") + slog.DebugContext(ctx, "use file referred; returning ENOENT for reference mgmt") return nil, syscall.ENOENT default: - log.G(ctx).Warnf("unknown filename %q", name) + slog.WarnContext(ctx, "unknown filename", "name", name) return nil, syscall.ENOENT } } diff --git a/pkg/fs/layer_node_more_test.go b/pkg/fs/layer_node_more_test.go new file mode 100644 index 0000000..6510cb7 --- /dev/null +++ b/pkg/fs/layer_node_more_test.go @@ -0,0 +1,18 @@ +package fs + +import ( + "context" + "testing" + + "github.com/hanwen/go-fuse/v2/fuse" +) + +func TestLayerNodeLookupUnknownNameENOENT(t *testing.T) { + n := &layerNode{fs: &fs{}} + var out fuse.EntryOut +_, eno := n.Lookup(context.Background(), "unknown", &out) + if eno == 0 { + t.Fatalf("expected ENOENT for unknown file name") + } +} + diff --git a/pkg/fs/layer_node_test.go b/pkg/fs/layer_node_test.go new file mode 100644 index 0000000..b0dc078 --- /dev/null +++ b/pkg/fs/layer_node_test.go @@ -0,0 +1,41 @@ +package fs + +import ( + "context" + "testing" + + "github.com/hanwen/go-fuse/v2/fuse" +) + +func TestLayerNodeReaddir(t *testing.T) { + n := &layerNode{} + ds, errno := n.Readdir(context.Background()) + if errno != 0 { + t.Fatalf("Readdir returned errno=%d", errno) + } + + var names []string + var modes []uint32 + for ds.HasNext() { + de, _ := ds.Next() + names = append(names, de.Name) + modes = append(modes, de.Mode) + } + + expectNames := []string{layerInfoLink, blobLink, layerLink, layerUseFile} + if len(names) != len(expectNames) { + t.Fatalf("unexpected entries len: got %d, want %d", len(names), len(expectNames)) + } + for i, want := range expectNames { + if names[i] != want { + t.Errorf("entry %d name mismatch: got %q want %q", i, names[i], want) + } + } + // modes + expectModes := []uint32{fuse.S_IFREG, fuse.S_IFREG, fuse.S_IFDIR, fuse.S_IFREG} + for i, want := range expectModes { + if modes[i] != want { + t.Errorf("entry %d mode mismatch: got %v want %v", i, modes[i], want) + } + } +} diff --git a/pkg/fs/ref_node.go b/pkg/fs/ref_node.go index ddd01bd..184d24b 100644 --- a/pkg/fs/ref_node.go +++ b/pkg/fs/ref_node.go @@ -5,8 +5,8 @@ package fs import ( "context" "syscall" + "log/slog" - "github.com/containerd/containerd/log" "github.com/containerd/containerd/reference" fusefs "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" @@ -28,13 +28,13 @@ var _ = (fusefs.NodeRmdirer)((*refNode)(nil)) func (n *refNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fusefs.Inode, syscall.Errno) { // lookup on memory nodes - log.L.WithContext(ctx).Debugf("ref node lookup name = %s", name) + slog.DebugContext(ctx, "ref node lookup", "name", name) if child := n.GetChild(name); child != nil { switch tn := child.Operations().(type) { case *layerNode: copyAttr(&out.Attr, &tn.attr) default: - log.G(ctx).Warn("rootnode.Lookup: uknown node type detected") + slog.WarnContext(ctx, "rootnode.Lookup: unknown node type detected") return nil, syscall.EIO } out.Attr.Ino = child.StableAttr().Ino @@ -42,7 +42,7 @@ func (n *refNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) ( } targetDigest, err := digest.Parse(name) if err != nil { - log.G(ctx).WithError(err).Errorf("invalid digest for %q", name) + slog.ErrorContext(ctx, "invalid digest", "name", name, "err", err) return nil, syscall.EINVAL } sAttr := defaultDirAttr(&out.Attr) @@ -63,12 +63,12 @@ func (n *refNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) ( func (n *refNode) Rmdir(ctx context.Context, name string) syscall.Errno { targetDigest, err := digest.Parse(name) if err != nil { - log.G(ctx).WithError(err).Warnf("invalid digest for %q during release", name) + slog.WarnContext(ctx, "invalid digest during release", "name", name, "err", err) return syscall.EINVAL } current, err := n.fs.layManager.Release(ctx, n.ref, targetDigest, n.rawRef) if err != nil { - log.G(ctx).WithError(err).Warnf("failed to release layer %v / %v", n.ref, targetDigest) + slog.WarnContext(ctx, "failed to release layer", "ref", n.ref.String(), "digest", targetDigest.String(), "err", err) return syscall.EIO } if current == 0 { @@ -76,7 +76,7 @@ func (n *refNode) Rmdir(ctx context.Context, name string) syscall.Errno { lh, ok := n.fs.knownNode[n.ref.String()][targetDigest.String()] if !ok { n.fs.knownNodeMu.Unlock() - log.G(ctx).WithError(err).Warnf("node of layer %v/%v is not registered", n.ref, targetDigest) + slog.WarnContext(ctx, "node of layer not registered", "ref", n.ref.String(), "digest", targetDigest.String(), "err", err) return syscall.EIO } lh.release() @@ -86,6 +86,6 @@ func (n *refNode) Rmdir(ctx context.Context, name string) syscall.Errno { } n.fs.knownNodeMu.Unlock() } - log.G(ctx).WithField("refcounter", current).Infof("layer %v/%v is marked as RELEASE", n.ref, targetDigest) + slog.InfoContext(ctx, "layer marked RELEASE", "ref", n.ref.String(), "digest", targetDigest.String(), "refcounter", current) return syscall.ENOENT } diff --git a/pkg/fs/ref_node_test.go b/pkg/fs/ref_node_test.go new file mode 100644 index 0000000..79a1d7b --- /dev/null +++ b/pkg/fs/ref_node_test.go @@ -0,0 +1,13 @@ +package fs + +import ( + "context" + "testing" +) + +func TestRefNodeRmdirInvalidDigest(t *testing.T) { + ref := refNode{fs: &fs{}} +if eno := ref.Rmdir(context.Background(), "not-a-digest"); eno == 0 { + t.Fatalf("expected error for invalid digest") + } +} diff --git a/pkg/fs/root_node.go b/pkg/fs/root_node.go index da38c1a..b5d358f 100644 --- a/pkg/fs/root_node.go +++ b/pkg/fs/root_node.go @@ -6,8 +6,8 @@ import ( "context" "encoding/base64" "syscall" + "log/slog" - "github.com/containerd/containerd/log" "github.com/containerd/containerd/reference" fusefs "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" @@ -26,7 +26,7 @@ var _ = (fusefs.NodeLookuper)((*rootNode)(nil)) // Lookup loads manifest and config of specified name (image reference) // and returns refnode of the specified name func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fusefs.Inode, syscall.Errno) { - log.L.WithContext(ctx).Debugf("root node lookup name = %s", name) + slog.DebugContext(ctx, "root node lookup", "name", name) if child := n.GetChild(name); child != nil { switch tn := child.Operations().(type) { case *fusefs.MemSymlink: @@ -34,7 +34,7 @@ func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) case *refNode: copyAttr(&out.Attr, &tn.attr) default: - log.L.WithContext(ctx).Warn("rootNode.Lookup: unknown node type detected") + slog.WarnContext(ctx, "rootNode.Lookup: unknown node type detected") return nil, syscall.EIO } out.Attr.Ino = child.StableAttr().Ino @@ -57,14 +57,14 @@ func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) refBytes, err := base64.StdEncoding.DecodeString(name) if err != nil { - log.G(ctx).WithError(err).Errorf("failed to decode ref base64 %q", name) + slog.ErrorContext(ctx, "failed to decode ref base64", "name", name, "err", err) return nil, syscall.EINVAL } ref := string(refBytes) var refSpec reference.Spec refSpec, err = reference.Parse(ref) if err != nil { - log.G(ctx).WithError(err).Errorf("invalid reference %q for %q", ref, name) + slog.ErrorContext(ctx, "invalid reference", "ref", ref, "raw", name, "err", err) return nil, syscall.EINVAL } sAttr := defaultDirAttr(&out.Attr) diff --git a/pkg/fs/root_node_test.go b/pkg/fs/root_node_test.go new file mode 100644 index 0000000..a7b9468 --- /dev/null +++ b/pkg/fs/root_node_test.go @@ -0,0 +1,3 @@ +package fs + +// Lookup paths depend on a running go-fuse server; covered by integration tests elsewhere. diff --git a/pkg/fs/utils_test.go b/pkg/fs/utils_test.go new file mode 100644 index 0000000..512fa0d --- /dev/null +++ b/pkg/fs/utils_test.go @@ -0,0 +1,48 @@ +package fs + +import ( + "testing" + + "github.com/hanwen/go-fuse/v2/fuse" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func TestUtilsCopyAttr(t *testing.T) { + var src, dst fuse.Attr + src.Ino = 123 + src.Size = 456 + src.Mode = 0o644 + src.Nlink = 2 + src.Owner = fuse.Owner{Uid: 1, Gid: 2} + copyAttr(&dst, &src) + if dst != src { + t.Fatalf("attr not copied: %+v != %+v", dst, src) + } +} + +func TestUtilsLayerToAttr(t *testing.T) { + var out fuse.Attr + s := layerToAttr(&ocispec.Descriptor{Size: 8192}, &out) + if out.Mode != layerFileMode { + t.Fatalf("unexpected mode: %o", out.Mode) + } + if s.Mode != out.Mode { + t.Fatalf("stable mode mismatch") + } +} + +func TestUtilsDefaultFileDirLinkAttr(t *testing.T) { + var out fuse.Attr + s1 := defaultFileAttr(100, &out) + if out.Mode != defaultFileMode || s1.Mode != out.Mode { + t.Fatalf("defaultFileAttr unexpected") + } + s2 := defaultDirAttr(&out) + if out.Mode != defaultDirMode || s2.Mode != out.Mode { + t.Fatalf("defaultDirAttr unexpected") + } + s3 := defaultLinkAttr(&out) + if out.Mode != defaultLinkMode || s3.Mode != out.Mode { + t.Fatalf("defaultLinkAttr unexpected") + } +} diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 4b54e58..c73d93c 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -7,8 +7,8 @@ import ( "path/filepath" "strings" "sync" + "log/slog" - "github.com/containerd/containerd/log" "github.com/containerd/containerd/reference" "github.com/containerd/containerd/snapshots/storage" "github.com/containerd/nydus-snapshotter/config" @@ -20,11 +20,21 @@ import ( "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" - "golang.org/x/sys/unix" "github.com/containers/nydus-storage-plugin/pkg/source" ) +// fsDriver abstracts the subset of nydus filesystem methods we use, to enable testing. +// Implemented by *nydusFS.Filesystem. +type fsDriver interface { + UpperPath(id string) string + PrepareMetaLayer(ctx context.Context, snapshot storage.Snapshot, annotations map[string]string) error + Mount(ctx context.Context, snapshotID string, annotations map[string]string) error + WaitUntilReady(ctx context.Context, snapshotID string) error + MountPoint(snapshotID string) (string, error) +} + + type nydusMessage struct { Err error } @@ -102,7 +112,7 @@ type LayerManager struct { refCounter map[string]map[string]int rootDir string nydusMetaLayer sync.Map - nydusFs *nydusFS.Filesystem + nydusFs fsDriver mu sync.Mutex } @@ -147,32 +157,32 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. layer.IsMetaLayer = true if _, ok = r.nydusMetaLayer.Load(snapshotID); ok { - log.G(ctx).Warnf("nydus duplicate mount meta layer ref is %s digest is %s", refspec.String(), target.Digest.String()) + slog.WarnContext(ctx, "nydus duplicate mount meta layer", "ref", refspec.String(), "digest", target.Digest.String()) return &layer, nil } workdir := r.nydusFs.UpperPath(snapshotID) if _, err = os.Stat(workdir); os.IsNotExist(err) { if err = os.MkdirAll(workdir, 0755); err != nil { - log.G(ctx).WithError(err).Error("mkdir nydus snapshot dir failed") + slog.ErrorContext(ctx, "mkdir nydus snapshot dir failed", "err", err) return &layer, err } } // Download nydus bootstrap layer to disk. - err = r.nydusFs.PrepareMetaLayer(ctx, storage.Snapshot{ID: snapshotID}, target.Annotations) - if err != nil && !strings.Contains(err.Error(), "file exists") { - log.G(ctx).WithError(err).Error("download snapshot files failed") - return &layer, err - } + err = r.nydusFs.PrepareMetaLayer(ctx, storage.Snapshot{ID: snapshotID}, target.Annotations) + if err != nil && !strings.Contains(err.Error(), "file exists") { + slog.ErrorContext(ctx, "download snapshot files failed", "err", err) + return &layer, err + } nydusMsgChannel := make(chan nydusMessage) go func() { - log.G(ctx).Debugf("nydus mount meta layer ref is %s digest is %s", refspec.String(), target.Digest.String()) + slog.DebugContext(ctx, "nydus mount meta layer", "ref", refspec.String(), "digest", target.Digest.String()) err = r.nydusFs.Mount(ctx, snapshotID, target.Annotations) if err != nil { - log.G(ctx).WithError(err).Error("nydus mount failed") + slog.ErrorContext(ctx, "nydus mount failed", "err", err) nydusMsgChannel <- nydusMessage{ Err: err, } @@ -199,24 +209,24 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. var mountPoint string if mountPoint, err = r.nydusFs.MountPoint(snapshotID); err == nil { if err := os.MkdirAll(targetPath, 0755); err != nil { - log.G(ctx).WithError(err).Error("ensure targetPath failed") + slog.ErrorContext(ctx, "ensure targetPath failed", "err", err) return &layer, err } // Perform a bind mount and then remount read-only for robustness across kernels - if err = unix.Mount(mountPoint, targetPath, "", unix.MS_BIND|unix.MS_REC, ""); err != nil { - log.G(ctx).WithError(err).Error("bind mount failed") +if err = unixMount(mountPoint, targetPath, "", msBind|msRec, ""); err != nil { + slog.ErrorContext(ctx, "bind mount failed", "err", err) return &layer, err } - if err = unix.Mount("", targetPath, "", unix.MS_BIND|unix.MS_REMOUNT|unix.MS_RDONLY, ""); err != nil { - log.G(ctx).WithError(err).Error("remount ro failed") +if err = unixMount("", targetPath, "", msBind|msRemount|msRdonly, ""); err != nil { + slog.ErrorContext(ctx, "remount ro failed", "err", err) // try to unmount in case remount failed partially - _ = unix.Unmount(targetPath, 0) + _ = unixUnmount(targetPath, 0) return &layer, err } r.nydusMetaLayer.Store(snapshotID, targetPath) return &layer, nil } - log.G(ctx).WithError(err).Error("get mount point failed") + slog.ErrorContext(ctx, "get mount point failed", "err", err) return &layer, err } // TODO support normal image format. @@ -240,8 +250,8 @@ func (r *LayerManager) Release(ctx context.Context, refspec reference.Spec, dgst i := r.refCounter[refspec.String()][dgst.String()] if i <= 0 { if v, ok := r.nydusMetaLayer.Load(snapshotID); ok { - if err := unix.Unmount(v.(string), 0); err != nil { - log.G(ctx).Errorf("umount bind nydus %v/%v failed: %+v", refspec, dgst, err) +if err := unixUnmount(v.(string), 0); err != nil { + slog.ErrorContext(ctx, "umount bind nydus failed", "ref", refspec.String(), "digest", dgst.String(), "err", err) return 0, err } r.nydusMetaLayer.Delete(snapshotID) @@ -251,20 +261,47 @@ func (r *LayerManager) Release(ctx context.Context, refspec reference.Spec, dgst if len(r.refCounter[refspec.String()]) == 0 { delete(r.refCounter, refspec.String()) } - log.G(ctx).WithField("refcounter", i).Infof("layer %v/%v is released due to no reference", refspec, dgst) + slog.InfoContext(ctx, "layer released due to no reference", "ref", refspec.String(), "digest", dgst.String(), "refcounter", i) } return i, nil } func (r *LayerManager) ReleaseAll(ctx context.Context) { r.nydusMetaLayer.Range(func(key, value interface{}) bool { - if err := unix.Unmount(value.(string), 0); err != nil { - log.G(ctx).WithError(err).Warnf("umount bind nydus %v/%v failed", key, value) +if err := unixUnmount(value.(string), 0); err != nil { + slog.WarnContext(ctx, "umount bind nydus failed", "key", key, "value", value, "err", err) } return true }) } +// RecoverOrphanMounts attempts to unmount any bind-mounted diff directories +// left over from previous crashes. It walks /store/*/*/diff and tries to unmount. +func (r *LayerManager) RecoverOrphanMounts(ctx context.Context) error { + storeRoot := filepath.Join(r.rootDir, "store") + ents, err := os.ReadDir(storeRoot) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + for _, e := range ents { + if !e.IsDir() { continue } + refDir := filepath.Join(storeRoot, e.Name()) + layers, _ := os.ReadDir(refDir) + for _, l := range layers { + if !l.IsDir() { continue } + diff := filepath.Join(refDir, l.Name(), "diff") + // Best-effort unmount + if err := unixUnmount(diff, 0); err == nil { + slog.DebugContext(ctx, "recovered orphan mount", "path", diff) + } + } + } + return nil +} + func (r *LayerManager) Use(refspec reference.Spec, dgst digest.Digest) int { r.refPool.use(refspec) diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go new file mode 100644 index 0000000..6a8fb36 --- /dev/null +++ b/pkg/manager/manager_test.go @@ -0,0 +1,185 @@ +package manager + +import ( + "context" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/containerd/containerd/reference" + "github.com/containerd/containerd/snapshots/storage" + "github.com/containerd/nydus-snapshotter/pkg/label" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +type fakeFS struct { + root string + mp string +} + +func (f *fakeFS) UpperPath(id string) string { return filepath.Join(f.root, "snapshots", id) } +func (f *fakeFS) PrepareMetaLayer(ctx context.Context, snapshot storage.Snapshot, annotations map[string]string) error { + return nil +} +func (f *fakeFS) Mount(ctx context.Context, snapshotID string, annotations map[string]string) error { return nil } +func (f *fakeFS) WaitUntilReady(ctx context.Context, snapshotID string) error { return nil } +func (f *fakeFS) MountPoint(snapshotID string) (string, error) { return f.mp, nil } + +func TestResolverMetaLayerBindMountAndRemountRO(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + + // prepare manifest+config in refPool + p, err := newRefPool(ctx, dir, nil) + if err != nil { + t.Fatalf("newRefPool: %v", err) + } + refspec, _ := reference.Parse("docker.io/library/busybox:latest") + dgst := digest.FromString("layer-1") + manifest := ocispec.Manifest{Layers: []ocispec.Descriptor{{ + MediaType: ocispec.MediaTypeImageLayerGzip, + Digest: dgst, + Size: 10, + Annotations: map[string]string{label.NydusMetaLayer: "true"}, + }}, Config: ocispec.Descriptor{MediaType: ocispec.MediaTypeImageConfig, Digest: digest.FromString("cfg"), Size: 1}} + config := ocispec.Image{RootFS: ocispec.RootFS{Type: "layers", DiffIDs: []digest.Digest{digest.FromString("diffid")}}} + if err := p.writeManifestAndConfig(refspec, manifest, config); err != nil { + t.Fatalf("writeManifestAndConfig: %v", err) + } + + mountPoint := filepath.Join(dir, "nydus-mp") + if err := os.MkdirAll(mountPoint, 0o755); err != nil { + t.Fatalf("mkdir mountpoint: %v", err) + } + + lm := &LayerManager{ + refPool: p, + refCounter: make(map[string]map[string]int), + nydusFs: &fakeFS{root: dir, mp: mountPoint}, + rootDir: dir, + } + + // stub unix mount + origMount := unixMount + defer func() { unixMount = origMount }() + type mcall struct{ src, tgt, fstype, data string; flags uintptr } + var calls []mcall + unixMount = func(source, target, fstype string, flags uintptr, data string) error { + calls = append(calls, mcall{source, target, fstype, data, flags}) + return nil + } + + snapshotID := "snap-1" + layer, err := lm.ResolverMetaLayer(ctx, refspec, snapshotID, dgst) + if err != nil { + t.Fatalf("ResolverMetaLayer error: %v", err) + } + if !layer.IsMetaLayer { + t.Fatalf("expected IsMetaLayer=true") + } + + targetPath := filepath.Join(dir, "store", snapshotID, dgst.String(), "diff") + if _, err := os.Stat(targetPath); err != nil { + t.Fatalf("targetPath not created: %v", err) + } + + if len(calls) != 2 { + t.Fatalf("expected 2 mount calls, got %d", len(calls)) + } + if calls[0].src != mountPoint || calls[0].tgt != targetPath { + t.Errorf("bind mount args mismatch: got (%q,%q), want (%q,%q)", calls[0].src, calls[0].tgt, mountPoint, targetPath) + } + if calls[1].src != "" || calls[1].tgt != targetPath { + t.Errorf("remount args mismatch: got (%q,%q)", calls[1].src, calls[1].tgt) + } + // flags + if calls[0].flags&(0x1000|0x4000) == 0 { // MS_BIND|MS_REC + t.Errorf("bind mount flags missing MS_BIND|MS_REC: %#x", calls[0].flags) + } + if calls[1].flags&(0x1000|0x20|0x1) == 0 { // MS_BIND|MS_REMOUNT|MS_RDONLY + t.Errorf("remount flags missing: %#x", calls[1].flags) + } +} + +func TestResolverMetaLayerCreatesTargetDir(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + p, err := newRefPool(ctx, dir, nil) + if err != nil { + t.Fatalf("newRefPool: %v", err) + } + refspec, _ := reference.Parse("docker.io/library/busybox:latest") + dgst := digest.FromString("layer-2") + manifest := ocispec.Manifest{Layers: []ocispec.Descriptor{{ + MediaType: ocispec.MediaTypeImageLayerGzip, + Digest: dgst, + Size: 10, + Annotations: map[string]string{label.NydusMetaLayer: "true"}, + }}, Config: ocispec.Descriptor{MediaType: ocispec.MediaTypeImageConfig, Digest: digest.FromString("cfg2"), Size: 1}} + config := ocispec.Image{RootFS: ocispec.RootFS{Type: "layers", DiffIDs: []digest.Digest{digest.FromString("diffid2")}}} + if err := p.writeManifestAndConfig(refspec, manifest, config); err != nil { + t.Fatalf("writeManifestAndConfig: %v", err) + } + mountPoint := filepath.Join(dir, "nydus-mp2") + if err := os.MkdirAll(mountPoint, 0o755); err != nil { + t.Fatalf("mkdir mountpoint: %v", err) + } + lm := &LayerManager{refPool: p, refCounter: make(map[string]map[string]int), nydusFs: &fakeFS{root: dir, mp: mountPoint}, rootDir: dir} + origMount := unixMount + defer func() { unixMount = origMount }() + unixMount = func(source, target, fstype string, flags uintptr, data string) error { return nil } + snapshotID := "snap-2" + _, err = lm.ResolverMetaLayer(ctx, refspec, snapshotID, dgst) + if err != nil { + t.Fatalf("ResolverMetaLayer error: %v", err) + } + targetPath := filepath.Join(dir, "store", snapshotID, dgst.String(), "diff") + if st, err := os.Stat(targetPath); err != nil || !st.IsDir() { + t.Fatalf("targetPath not created as dir: %v, st=%v", err, st) + } +} + +func TestReleaseDecrementsAndUnmountsAndCleansMaps(t *testing.T) { + ctx := context.Background() + lm := &LayerManager{ + refPool: &refPool{refcounter: map[string]*releaser{}}, + refCounter: map[string]map[string]int{}, + } + refspec, _ := reference.Parse("docker.io/library/busybox:latest") + dgst := digest.FromString("layer-1") + snapshotID := "snap-1" + // prepare counters + lm.refCounter[refspec.String()] = map[string]int{dgst.String(): 1} + lm.nydusMetaLayer.Store(snapshotID, "/fake/target") + lm.refPool.refcounter[refspec.String()] = &releaser{count: 1, release: func() {}} + + var unmounted []string + origUnmount := unixUnmount + defer func() { unixUnmount = origUnmount }() + unixUnmount = func(target string, flags int) error { + unmounted = append(unmounted, target) + return nil + } + + i, err := lm.Release(ctx, refspec, dgst, snapshotID) + if err != nil { + t.Fatalf("Release error: %v", err) + } + if i != 0 { + t.Fatalf("expected return 0, got %d", i) + } + if !reflect.DeepEqual(unmounted, []string{"/fake/target"}) { + t.Errorf("unexpected unmounts: %#v", unmounted) + } + if _, ok := lm.refCounter[refspec.String()][dgst.String()]; ok { + t.Errorf("layer entry not removed from refCounter") + } + if _, ok := lm.refCounter[refspec.String()]; ok { + t.Errorf("ref entry not removed from refCounter") + } + if _, ok := lm.nydusMetaLayer.Load(snapshotID); ok { + t.Errorf("nydusMetaLayer entry not deleted") + } +} diff --git a/pkg/manager/mount_shim_linux.go b/pkg/manager/mount_shim_linux.go new file mode 100644 index 0000000..6870a69 --- /dev/null +++ b/pkg/manager/mount_shim_linux.go @@ -0,0 +1,24 @@ +//go:build linux + +package manager + +import "golang.org/x/sys/unix" + +const ( + msBind = unix.MS_BIND + msRec = unix.MS_REC + msRemount = unix.MS_REMOUNT + msRdonly = unix.MS_RDONLY +) + +type mountFunc func(source, target, fstype string, flags uintptr, data string) error + +type unmountFunc func(target string, flags int) error + +var unixMount mountFunc = func(source, target, fstype string, flags uintptr, data string) error { + return unix.Mount(source, target, fstype, flags, data) +} + +var unixUnmount unmountFunc = func(target string, flags int) error { + return unix.Unmount(target, flags) +} diff --git a/pkg/manager/mount_shim_other.go b/pkg/manager/mount_shim_other.go new file mode 100644 index 0000000..9c1885a --- /dev/null +++ b/pkg/manager/mount_shim_other.go @@ -0,0 +1,19 @@ +//go:build !linux + +package manager + +const ( + // Linux mount flags used for tests; values chosen to match linux for assertions. + msBind = 0x1000 + msRec = 0x4000 + msRemount = 0x20 + msRdonly = 0x1 +) + +type mountFunc func(source, target, fstype string, flags uintptr, data string) error + +type unmountFunc func(target string, flags int) error + +// On non-linux, provide stubs to satisfy compilation; tests override these when needed. +var unixMount mountFunc = func(source, target, fstype string, flags uintptr, data string) error { return nil } +var unixUnmount unmountFunc = func(target string, flags int) error { return nil } diff --git a/pkg/manager/ref_pool.go b/pkg/manager/ref_pool.go index 16ca6ce..d1d31d2 100644 --- a/pkg/manager/ref_pool.go +++ b/pkg/manager/ref_pool.go @@ -11,9 +11,9 @@ import ( "path/filepath" "sync" "time" + "log/slog" "github.com/containerd/containerd/images" - "github.com/containerd/containerd/log" "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/reference" "github.com/containerd/containerd/remotes" @@ -45,10 +45,10 @@ func newRefPool(ctx context.Context, root string, hosts source.RegistryHosts) (* p.cache.OnEvicted = func(key string, value interface{}) { refspec := value.(reference.Spec) if err := os.RemoveAll(p.metadataDir(refspec)); err != nil { - log.G(ctx).WithField("key", key).WithError(err).Warnf("failed to clean up ref") + slog.WarnContext(ctx, "failed to clean up ref", "key", key, "err", err) return } - log.G(ctx).WithField("key", key).Debugf("cleaned up ref") + slog.DebugContext(ctx, "cleaned up ref", "key", key) } return p, nil } @@ -70,10 +70,10 @@ type releaser struct { func (p *refPool) loadRef(ctx context.Context, refspec reference.Spec) (manifest ocispec.Manifest, config ocispec.Image, err error) { manifest, config, err = p.readManifestAndConfig(refspec) if err == nil { - log.G(ctx).Debugf("reusing manifest and config of %q", refspec.String()) + slog.DebugContext(ctx, "reusing manifest and config", "ref", refspec.String()) return } - log.G(ctx).WithError(err).Debugf("fetching manifest and config of %q", refspec.String()) + slog.DebugContext(ctx, "fetching manifest and config", "ref", refspec.String(), "err", err) manifest, config, err = p.fetchManifestAndConfig(ctx, refspec) if err != nil { return ocispec.Manifest{}, ocispec.Image{}, err @@ -152,7 +152,7 @@ func (p *refPool) readManifestAndConfig(refspec reference.Spec) (manifest ocispe func (p *refPool) writeManifestAndConfig(refspec reference.Spec, manifest ocispec.Manifest, config ocispec.Image) error { mPath, cPath := p.manifestFile(refspec), p.configFile(refspec) - log.G(context.TODO()).Infof("mpath = %s, cpath = %s", mPath, cPath) + slog.Info("write manifest and config paths", "manifest", mPath, "config", cPath) if err := os.MkdirAll(filepath.Dir(mPath), 0700); err != nil { return err } diff --git a/pkg/services/keychain/dockerconfig/dockerconfig.go b/pkg/services/keychain/dockerconfig/dockerconfig.go index d657d5f..ef145d3 100644 --- a/pkg/services/keychain/dockerconfig/dockerconfig.go +++ b/pkg/services/keychain/dockerconfig/dockerconfig.go @@ -2,8 +2,8 @@ package dockerconfig import ( "context" + "log/slog" - "github.com/containerd/containerd/log" "github.com/containerd/containerd/reference" "github.com/docker/cli/cli/config" @@ -16,7 +16,7 @@ func NewDockerconfigKeychain(ctx context.Context) resolver.Credential { return func(host string, refspec reference.Spec) (string, string, error) { cf, err := config.Load("") if err != nil { - log.G(ctx).WithError(err).Warnf("failed to load docker config file") + slog.WarnContext(ctx, "failed to load docker config file", "err", err) return "", "", nil } diff --git a/pkg/services/keychain/podmanauth/podmanauth.go b/pkg/services/keychain/podmanauth/podmanauth.go new file mode 100644 index 0000000..49fafcf --- /dev/null +++ b/pkg/services/keychain/podmanauth/podmanauth.go @@ -0,0 +1,110 @@ +package podmanauth + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/reference" + + "github.com/containers/nydus-storage-plugin/pkg/services/resolver" +) + +// NewPodmanAuthKeychain returns a resolver.Credential that sources credentials +// from Podman-compatible auth.json files. +// Precedence: +// 1) REGISTRY_AUTH_FILE (path to an auth.json) +// 2) $XDG_RUNTIME_DIR/containers/auth.json +// 3) $HOME/.config/containers/auth.json +func NewPodmanAuthKeychain(ctx context.Context) resolver.Credential { + return func(host string, refspec reference.Spec) (string, string, error) { + path, err := findAuthFile() + if err != nil || path == "" { + return "", "", nil + } + creds, err := readAuthFile(path) + if err != nil { + log.G(ctx).WithError(err).Warnf("failed to read podman auth.json from %s", path) + return "", "", nil + } + + // Docker Hub special-case compatibility + if host == "docker.io" || host == "registry-1.docker.io" { + host = "https://index.docker.io/v1/" + } + + if e, ok := creds.Auths[host]; ok { + if e.IdentityToken != "" { + return "", e.IdentityToken, nil + } + // Prefer explicit username/password if present + if e.Username != "" || e.Password != "" { + return e.Username, e.Password, nil + } + if e.Auth != "" { + b, err := base64.StdEncoding.DecodeString(e.Auth) + if err == nil { + p := string(b) + if idx := strings.IndexByte(p, ':'); idx >= 0 { + return p[:idx], p[idx+1:], nil + } + } + } + } + return "", "", nil + } +} + +// Minimal struct for containers-auth.json +type authFile struct { + Auths map[string]authEntry `json:"auths"` +} + +type authEntry struct { + Auth string `json:"auth"` + Username string `json:"username"` + Password string `json:"password"` + IdentityToken string `json:"identitytoken"` +} + +func findAuthFile() (string, error) { + if p := os.Getenv("REGISTRY_AUTH_FILE"); p != "" { + return p, nil + } + if xdg := os.Getenv("XDG_RUNTIME_DIR"); xdg != "" { + p := filepath.Join(xdg, "containers", "auth.json") + if _, err := os.Stat(p); err == nil { + return p, nil + } + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + p := filepath.Join(home, ".config", "containers", "auth.json") + if _, err := os.Stat(p); err == nil { + return p, nil + } + return "", nil +} + +func readAuthFile(path string) (authFile, error) { + f, err := os.Open(path) + if err != nil { + return authFile{}, err + } + defer f.Close() + var af authFile + if err := json.NewDecoder(f).Decode(&af); err != nil { + return authFile{}, err + } + if af.Auths == nil { + return authFile{}, errors.New("no auths") + } + return af, nil +} \ No newline at end of file From eda3a1aa8cf480435567a6ff296de4f7424b5ca8 Mon Sep 17 00:00:00 2001 From: fatelei Date: Sun, 26 Oct 2025 22:56:03 +0800 Subject: [PATCH 06/23] chore: upgrade go version Signed-off-by: fatelei --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 923851f..9bdb887 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.18' + go-version: '1.25' - name: Login ghcr uses: docker/login-action@v3 with: From 956d9dbae1ecf70a0748d5249a082929c9248232 Mon Sep 17 00:00:00 2001 From: fatelei Date: Tue, 28 Oct 2025 11:14:33 +0800 Subject: [PATCH 07/23] fix: fix lint error Signed-off-by: fatelei --- .github/workflows/ci.yml | 4 +- .golangci.yml | 41 ++++++----- cmd/store/main.go | 46 ++++++++---- pkg/cache/lrucache.go | 2 +- pkg/fs/blob_file.go | 4 +- pkg/fs/blob_file_test.go | 4 +- pkg/fs/blob_node.go | 2 +- pkg/fs/blob_node_test.go | 2 +- pkg/fs/diff_node.go | 2 +- pkg/fs/diff_node_test.go | 4 +- pkg/fs/fs.go | 2 +- pkg/fs/id_map.go | 6 +- pkg/fs/id_map_test.go | 2 +- pkg/fs/layer_node.go | 20 ++--- pkg/fs/layer_node_more_test.go | 3 +- pkg/fs/ref_node.go | 6 +- pkg/fs/ref_node_test.go | 2 +- pkg/fs/root_node.go | 12 +-- pkg/manager/manager.go | 73 ++++++++++--------- pkg/manager/manager_test.go | 19 +++-- pkg/manager/mount_shim_other.go | 4 +- pkg/manager/ref_pool.go | 8 +- .../keychain/dockerconfig/dockerconfig.go | 2 +- .../keychain/podmanauth/podmanauth.go | 4 +- pkg/services/resolver/resolver.go | 2 +- pkg/utils/manifest.go | 2 +- 26 files changed, 153 insertions(+), 125 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9bdb887..090dca4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.25' + go-version: "1.25" - name: Login ghcr uses: docker/login-action@v3 with: @@ -26,7 +26,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build run: | - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.48.0 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0 export PATH=$PATH:$(go env GOPATH)/bin make check make build diff --git a/.golangci.yml b/.golangci.yml index 19981df..93ca81a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,22 +1,29 @@ -# https://golangci-lint.run/usage/configuration#config-file - +version: "2" linters: enable: - - structcheck - - varcheck - - staticcheck - - unconvert - - gofmt - - goimports - - revive - - ineffassign - - vet - - unused - misspell + - revive + - unconvert disable: - errcheck - -run: - deadline: 4m - skip-dirs: - - misc + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - gofmt + - goimports + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/cmd/store/main.go b/cmd/store/main.go index fe4bd1f..8a46b38 100644 --- a/cmd/store/main.go +++ b/cmd/store/main.go @@ -2,11 +2,11 @@ package main import ( "fmt" + "log/slog" "os" "os/signal" - "syscall" - "log/slog" "path/filepath" + "syscall" "github.com/containerd/nydus-snapshotter/cmd/containerd-nydus-grpc/pkg/command" "github.com/containerd/nydus-snapshotter/config" @@ -19,7 +19,7 @@ import ( "github.com/containers/nydus-storage-plugin/pkg/services/keychain/dockerconfig" podmanauth "github.com/containers/nydus-storage-plugin/pkg/services/keychain/podmanauth" "github.com/containers/nydus-storage-plugin/pkg/services/resolver" - ) +) func waitForSIGINT() { c := make(chan os.Signal, 1) @@ -31,19 +31,25 @@ func parseOctalMode(s string) (uint32, error) { var v uint32 for i := 0; i < len(s); i++ { c := s[i] - if c < '0' || c > '7' { return 0, fmt.Errorf("invalid octal: %s", s) } + if c < '0' || c > '7' { + return 0, fmt.Errorf("invalid octal: %s", s) + } v = (v << 3) | uint32(c-'0') } return v, nil } -func setupSlog(level string, toStdout bool, logDir string, root string) error { +func setupSlog(level string, toStdout bool, logDir string) error { var lvl slog.Level switch level { - case "debug": lvl = slog.LevelDebug - case "warn", "warning": lvl = slog.LevelWarn - case "error": lvl = slog.LevelError - default: lvl = slog.LevelInfo + case "debug": + lvl = slog.LevelDebug + case "warn", "warning": + lvl = slog.LevelWarn + case "error": + lvl = slog.LevelError + default: + lvl = slog.LevelInfo } var w *os.File if toStdout || logDir == "" { @@ -51,7 +57,9 @@ func setupSlog(level string, toStdout bool, logDir string, root string) error { } else { path := filepath.Join(logDir, "nydus-store.log") f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if err != nil { return err } + if err != nil { + return err + } w = f } h := slog.NewTextHandler(w, &slog.HandlerOptions{Level: lvl}) @@ -65,13 +73,13 @@ func main() { Name: "crio nydus store", Usage: "crio nydus store plugin", Version: "0.0.0", - Flags: append(flags.F, + Flags: append(flags.F, &cli.StringFlag{Name: "fs-file-mode", Usage: "octal file mode for files (e.g. 0400)"}, &cli.StringFlag{Name: "fs-dir-mode", Usage: "octal dir mode (e.g. 0500)"}, &cli.StringFlag{Name: "fs-link-mode", Usage: "octal symlink mode (e.g. 0400)"}, ), Action: func(c *cli.Context) error { - if err := setupSlog(flags.Args.LogLevel, flags.Args.LogToStdout, flags.Args.LogDir, flags.Args.RootDir); err != nil { + if err := setupSlog(flags.Args.LogLevel, flags.Args.LogToStdout, flags.Args.LogDir); err != nil { return errors.Wrap(err, "failed to prepare logger") } @@ -91,7 +99,7 @@ func main() { dockerconfig.NewDockerconfigKeychain(c.Context), // Podman-compatible auth.json podmanauth.NewPodmanAuthKeychain(c.Context), - }..., + }..., ) layManager, err := manager.NewLayerManager(c.Context, flags.Args.RootDir, hosts, &cfg) if err != nil { @@ -103,13 +111,19 @@ func main() { dirMode := fs.DefaultDirMode() linkMode := fs.DefaultLinkMode() if v := c.String("fs-file-mode"); v != "" { - if m, err := parseOctalMode(v); err == nil { fileMode = m } + if m, err := parseOctalMode(v); err == nil { + fileMode = m + } } if v := c.String("fs-dir-mode"); v != "" { - if m, err := parseOctalMode(v); err == nil { dirMode = m } + if m, err := parseOctalMode(v); err == nil { + dirMode = m + } } if v := c.String("fs-link-mode"); v != "" { - if m, err := parseOctalMode(v); err == nil { linkMode = m } + if m, err := parseOctalMode(v); err == nil { + linkMode = m + } } // Recover orphan bind mounts from previous crashes diff --git a/pkg/cache/lrucache.go b/pkg/cache/lrucache.go index 9fc1eee..e7a909b 100644 --- a/pkg/cache/lrucache.go +++ b/pkg/cache/lrucache.go @@ -23,7 +23,7 @@ type LRUCache struct { // NewLRUCache creates new lru cache. func NewLRUCache(maxEntries int) *LRUCache { inner := lru.New(maxEntries) - inner.OnEvicted = func(key lru.Key, value interface{}) { + inner.OnEvicted = func(_ lru.Key, value interface{}) { // Decrease the ref count incremented in Add(). // When nobody refers to this value, this value will be finalized via refCounter. value.(*refCounter).finalize() diff --git a/pkg/fs/blob_file.go b/pkg/fs/blob_file.go index 7a7aa74..769327d 100644 --- a/pkg/fs/blob_file.go +++ b/pkg/fs/blob_file.go @@ -14,7 +14,7 @@ type blobFile struct { var _ = (fusefs.FileReader)((*blobFile)(nil)) -func (f *blobFile) Read(ctx context.Context, dest []byte, off int64) (fuse.ReadResult, syscall.Errno) { +func (f *blobFile) Read(_ context.Context, _ []byte, _ int64) (fuse.ReadResult, syscall.Errno) { //s, err := f.l.ReadAt(dest, off, // remote.WithContext(ctx), // Make cancellable // remote.WithCacheOpts(cache.Direct()), // Do not pollute mem cache @@ -28,7 +28,7 @@ func (f *blobFile) Read(ctx context.Context, dest []byte, off int64) (fuse.ReadR var _ = (fusefs.FileGetattrer)((*blobFile)(nil)) -func (f *blobFile) Getattr(ctx context.Context, out *fuse.AttrOut) syscall.Errno { +func (f *blobFile) Getattr(_ context.Context, _ *fuse.AttrOut) syscall.Errno { //layerToAttr(f.l, &out.Attr) return 0 } diff --git a/pkg/fs/blob_file_test.go b/pkg/fs/blob_file_test.go index cf82104..a432c07 100644 --- a/pkg/fs/blob_file_test.go +++ b/pkg/fs/blob_file_test.go @@ -9,7 +9,7 @@ import ( func TestBlobFileReadEIO(t *testing.T) { bf := &blobFile{} -if rr, eno := bf.Read(context.Background(), nil, 0); eno == 0 || rr != nil { + if rr, eno := bf.Read(context.Background(), nil, 0); eno == 0 || rr != nil { t.Fatalf("expected EIO and nil read result, got eno=%d rr=%v", eno, rr) } } @@ -17,7 +17,7 @@ if rr, eno := bf.Read(context.Background(), nil, 0); eno == 0 || rr != nil { func TestBlobFileGetattrNoError(t *testing.T) { bf := &blobFile{} var out fuse.AttrOut -if eno := bf.Getattr(context.Background(), &out); eno != 0 { + if eno := bf.Getattr(context.Background(), &out); eno != 0 { t.Fatalf("Getattr returned errno=%d", eno) } } diff --git a/pkg/fs/blob_node.go b/pkg/fs/blob_node.go index d3e8ffd..52708bc 100644 --- a/pkg/fs/blob_node.go +++ b/pkg/fs/blob_node.go @@ -20,6 +20,6 @@ var _ = (fusefs.InodeEmbedder)((*blobNode)(nil)) var _ = (fusefs.NodeOpener)((*blobNode)(nil)) -func (n *blobNode) Open(ctx context.Context, flags uint32) (fh fusefs.FileHandle, fuseFlags uint32, errno syscall.Errno) { +func (n *blobNode) Open(_ context.Context, _ uint32) (fh fusefs.FileHandle, fuseFlags uint32, errno syscall.Errno) { return &blobFile{}, 0, 0 } diff --git a/pkg/fs/blob_node_test.go b/pkg/fs/blob_node_test.go index f5509f9..3a7eb93 100644 --- a/pkg/fs/blob_node_test.go +++ b/pkg/fs/blob_node_test.go @@ -7,7 +7,7 @@ import ( func TestBlobNodeOpenReturnsBlobFile(t *testing.T) { n := &blobNode{} -fh, _, eno := n.Open(context.Background(), 0) + fh, _, eno := n.Open(context.Background(), 0) if eno != 0 { t.Fatalf("Open errno=%d", eno) } diff --git a/pkg/fs/diff_node.go b/pkg/fs/diff_node.go index 9774b4c..7798bd6 100644 --- a/pkg/fs/diff_node.go +++ b/pkg/fs/diff_node.go @@ -15,7 +15,7 @@ type diffNode struct { fs *fs } -func (n *diffNode) Getattr(ctx context.Context, f fusefs.FileHandle, out *fuse.AttrOut) syscall.Errno { +func (n *diffNode) Getattr(_ context.Context, _ fusefs.FileHandle, out *fuse.AttrOut) syscall.Errno { copyAttr(&out.Attr, &n.attr) return 0 } diff --git a/pkg/fs/diff_node_test.go b/pkg/fs/diff_node_test.go index 3f126f6..26716a2 100644 --- a/pkg/fs/diff_node_test.go +++ b/pkg/fs/diff_node_test.go @@ -14,8 +14,8 @@ func TestDiffNodeGetattrCopiesAttr(t *testing.T) { if eno := d.Getattr(context.Background(), nil, &out); eno != 0 { t.Fatalf("Getattr errno=%d", eno) } - if out.Attr.Mode != d.attr.Mode { - t.Fatalf("mode mismatch: got %o want %o", out.Attr.Mode, d.attr.Mode) + if out.Mode != d.attr.Mode { + t.Fatalf("mode mismatch: got %o want %o", out.Mode, d.attr.Mode) } } diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 62cf7d4..3d7ae1b 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -99,7 +99,7 @@ func (r *inoReleasable) releasable() bool { return r.n.EmbeddedInode().Forgotten() } -func Mount(ctx context.Context, mountPoint string, rootDir string, debug bool, layManager *manager.LayerManager, opts ...MountOption) error { +func Mount(_ context.Context, mountPoint string, _ string, debug bool, layManager *manager.LayerManager, opts ...MountOption) error { // Apply mount options for _, o := range opts { if o != nil { diff --git a/pkg/fs/id_map.go b/pkg/fs/id_map.go index b3b646a..f6ebfd3 100644 --- a/pkg/fs/id_map.go +++ b/pkg/fs/id_map.go @@ -24,17 +24,17 @@ func (m *idMap) add(p func(uint32) (releasable, error)) error { m.cleanupG.Do("cleanup", func() (interface{}, error) { m.mu.Lock() defer m.mu.Unlock() - max := uint32(0) + maxID := uint32(0) for i := uint32(0); i <= m.max; i++ { if e, ok := m.m[i]; ok { if e.releasable() { delete(m.m, i) } else { - max = i + maxID = i } } } - m.max = max + m.max = maxID return nil, nil }) diff --git a/pkg/fs/id_map_test.go b/pkg/fs/id_map_test.go index c7a837a..cc82138 100644 --- a/pkg/fs/id_map_test.go +++ b/pkg/fs/id_map_test.go @@ -11,7 +11,7 @@ func (r *tr) releasable() bool { return r.releasableFlag } func TestIDMapAddAssignsAndReuses(t *testing.T) { m := &idMap{} var saved []*tr - add := func(id uint32) (releasable, error) { + add := func(_ uint32) (releasable, error) { r := &tr{} saved = append(saved, r) return r, nil diff --git a/pkg/fs/layer_node.go b/pkg/fs/layer_node.go index c6cfa32..acca5df 100644 --- a/pkg/fs/layer_node.go +++ b/pkg/fs/layer_node.go @@ -6,8 +6,8 @@ import ( "bytes" "context" "encoding/json" - "syscall" "log/slog" + "syscall" fusefs "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" @@ -31,7 +31,7 @@ var _ = (fusefs.NodeReaddirer)((*layerNode)(nil)) // Create marks this layer as "using". // We don't use refnode.Mkdir because Mkdir event doesn't reach here if layernode already exists. -func (n *layerNode) Create(ctx context.Context, name string, flags uint32, mode uint32, out *fuse.EntryOut) (node *fusefs.Inode, fh fusefs.FileHandle, fuseFlags uint32, errno syscall.Errno) { +func (n *layerNode) Create(ctx context.Context, name string, _ uint32, _ uint32, _ *fuse.EntryOut) (node *fusefs.Inode, fh fusefs.FileHandle, fuseFlags uint32, errno syscall.Errno) { if name == layerUseFile { current := n.fs.layManager.Use(n.refNode.ref, n.digest) slog.InfoContext(ctx, "layer marked USING", "ref", n.refNode.ref.String(), "digest", n.digest.String(), "refcounter", current) @@ -59,7 +59,7 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) cn := &fusefs.MemRegularFile{Data: infoData} copyAttr(&cn.Attr, &out.Attr) return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { - out.Attr.Ino = uint64(ino) + out.Ino = uint64(ino) cn.Attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) return n.NewInode(ctx, cn, sAttr) @@ -75,8 +75,8 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) copyAttr(&out.Attr, &ao.Attr) n.fs.knownNodeMu.Unlock() return n.NewInode(ctx, lh.n, fusefs.StableAttr{ - Mode: out.Attr.Mode, - Ino: out.Attr.Ino, + Mode: out.Mode, + Ino: out.Ino, }), 0 } n.fs.knownNodeMu.Unlock() @@ -94,7 +94,7 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) cn := &blobNode{l: &l.Descriptor} copyAttr(&cn.attr, &out.Attr) return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { - out.Attr.Ino = uint64(ino) + out.Ino = uint64(ino) cn.attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) return n.NewInode(ctx, cn, sAttr) @@ -113,7 +113,7 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) } copyAttr(&child.attr, &out.Attr) return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { - out.Attr.Ino = uint64(ino) + out.Ino = uint64(ino) child.attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) cn := n.NewInode(ctx, child, sAttr) @@ -131,16 +131,16 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) return cn }) case layerUseFile: - slog.DebugContext(ctx, "use file referred; returning ENOENT for reference mgmt") + slog.DebugContext(ctx, "use file referred; returning ENOENT for reference mgmt") return nil, syscall.ENOENT default: - slog.WarnContext(ctx, "unknown filename", "name", name) + slog.WarnContext(ctx, "unknown filename", "name", name) return nil, syscall.ENOENT } } // Readdir enumerates expected entries to help consumers like Podman discover files reliably. -func (n *layerNode) Readdir(ctx context.Context) (fusefs.DirStream, syscall.Errno) { +func (n *layerNode) Readdir(_ context.Context) (fusefs.DirStream, syscall.Errno) { entries := []fuse.DirEntry{ {Name: layerInfoLink, Mode: fuse.S_IFREG}, {Name: blobLink, Mode: fuse.S_IFREG}, diff --git a/pkg/fs/layer_node_more_test.go b/pkg/fs/layer_node_more_test.go index 6510cb7..3bdf322 100644 --- a/pkg/fs/layer_node_more_test.go +++ b/pkg/fs/layer_node_more_test.go @@ -10,9 +10,8 @@ import ( func TestLayerNodeLookupUnknownNameENOENT(t *testing.T) { n := &layerNode{fs: &fs{}} var out fuse.EntryOut -_, eno := n.Lookup(context.Background(), "unknown", &out) + _, eno := n.Lookup(context.Background(), "unknown", &out) if eno == 0 { t.Fatalf("expected ENOENT for unknown file name") } } - diff --git a/pkg/fs/ref_node.go b/pkg/fs/ref_node.go index 184d24b..366ddc3 100644 --- a/pkg/fs/ref_node.go +++ b/pkg/fs/ref_node.go @@ -4,8 +4,8 @@ package fs import ( "context" - "syscall" "log/slog" + "syscall" "github.com/containerd/containerd/reference" fusefs "github.com/hanwen/go-fuse/v2/fs" @@ -37,7 +37,7 @@ func (n *refNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) ( slog.WarnContext(ctx, "rootnode.Lookup: unknown node type detected") return nil, syscall.EIO } - out.Attr.Ino = child.StableAttr().Ino + out.Ino = child.StableAttr().Ino return child, 0 } targetDigest, err := digest.Parse(name) @@ -53,7 +53,7 @@ func (n *refNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) ( } copyAttr(&child.attr, &out.Attr) return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { - out.Attr.Ino = uint64(ino) + out.Ino = uint64(ino) child.attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) return n.NewInode(ctx, child, sAttr) diff --git a/pkg/fs/ref_node_test.go b/pkg/fs/ref_node_test.go index 79a1d7b..9c9d15b 100644 --- a/pkg/fs/ref_node_test.go +++ b/pkg/fs/ref_node_test.go @@ -7,7 +7,7 @@ import ( func TestRefNodeRmdirInvalidDigest(t *testing.T) { ref := refNode{fs: &fs{}} -if eno := ref.Rmdir(context.Background(), "not-a-digest"); eno == 0 { + if eno := ref.Rmdir(context.Background(), "not-a-digest"); eno == 0 { t.Fatalf("expected error for invalid digest") } } diff --git a/pkg/fs/root_node.go b/pkg/fs/root_node.go index b5d358f..3a13301 100644 --- a/pkg/fs/root_node.go +++ b/pkg/fs/root_node.go @@ -5,8 +5,8 @@ package fs import ( "context" "encoding/base64" - "syscall" "log/slog" + "syscall" "github.com/containerd/containerd/reference" fusefs "github.com/hanwen/go-fuse/v2/fs" @@ -37,7 +37,7 @@ func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) slog.WarnContext(ctx, "rootNode.Lookup: unknown node type detected") return nil, syscall.EIO } - out.Attr.Ino = child.StableAttr().Ino + out.Ino = child.StableAttr().Ino return child, 0 } @@ -48,7 +48,7 @@ func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) cn := &fusefs.MemSymlink{Data: []byte(n.fs.layManager.RefRoot())} copyAttr(&cn.Attr, &out.Attr) return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { - out.Attr.Ino = uint64(ino) + out.Ino = uint64(ino) cn.Attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) return n.NewInode(ctx, cn, sAttr) @@ -57,14 +57,14 @@ func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) refBytes, err := base64.StdEncoding.DecodeString(name) if err != nil { - slog.ErrorContext(ctx, "failed to decode ref base64", "name", name, "err", err) + slog.ErrorContext(ctx, "failed to decode ref base64", "name", name, "err", err) return nil, syscall.EINVAL } ref := string(refBytes) var refSpec reference.Spec refSpec, err = reference.Parse(ref) if err != nil { - slog.ErrorContext(ctx, "invalid reference", "ref", ref, "raw", name, "err", err) + slog.ErrorContext(ctx, "invalid reference", "ref", ref, "raw", name, "err", err) return nil, syscall.EINVAL } sAttr := defaultDirAttr(&out.Attr) @@ -75,7 +75,7 @@ func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) } copyAttr(&child.attr, &out.Attr) return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { - out.Attr.Ino = uint64(ino) + out.Ino = uint64(ino) child.attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) return n.NewInode(ctx, child, sAttr) diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index c73d93c..e5bb9e1 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -3,11 +3,11 @@ package manager import ( "context" "fmt" + "log/slog" "os" "path/filepath" "strings" "sync" - "log/slog" "github.com/containerd/containerd/reference" "github.com/containerd/containerd/snapshots/storage" @@ -34,7 +34,6 @@ type fsDriver interface { MountPoint(snapshotID string) (string, error) } - type nydusMessage struct { Err error } @@ -45,7 +44,7 @@ type MountLayer struct { MountFailed bool } -var MountMetaLayerFailed = errors.New("mount meta layer failed") +var ErrMountMetaLayerFailed = errors.New("mount meta layer failed") func NewLayerManager(ctx context.Context, rootDir string, hosts source.RegistryHosts, cfg *config.Config) (*LayerManager, error) { verifier, err := signature.NewVerifier(cfg.PublicKeyFile, cfg.ValidateSignature) @@ -170,11 +169,11 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. } // Download nydus bootstrap layer to disk. - err = r.nydusFs.PrepareMetaLayer(ctx, storage.Snapshot{ID: snapshotID}, target.Annotations) - if err != nil && !strings.Contains(err.Error(), "file exists") { - slog.ErrorContext(ctx, "download snapshot files failed", "err", err) - return &layer, err - } + err = r.nydusFs.PrepareMetaLayer(ctx, storage.Snapshot{ID: snapshotID}, target.Annotations) + if err != nil && !strings.Contains(err.Error(), "file exists") { + slog.ErrorContext(ctx, "download snapshot files failed", "err", err) + return &layer, err + } nydusMsgChannel := make(chan nydusMessage) @@ -201,33 +200,33 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. err = r.nydusFs.WaitUntilReady(ctx, snapshotID) if err != nil { - return &layer, MountMetaLayerFailed + return &layer, ErrMountMetaLayerFailed } // Link nydusd mount dir to ///diff - targetPath := filepath.Join(r.rootDir, "store", snapshotID, target.Digest.String(), "diff") - var mountPoint string - if mountPoint, err = r.nydusFs.MountPoint(snapshotID); err == nil { - if err := os.MkdirAll(targetPath, 0755); err != nil { - slog.ErrorContext(ctx, "ensure targetPath failed", "err", err) - return &layer, err - } - // Perform a bind mount and then remount read-only for robustness across kernels -if err = unixMount(mountPoint, targetPath, "", msBind|msRec, ""); err != nil { - slog.ErrorContext(ctx, "bind mount failed", "err", err) - return &layer, err - } -if err = unixMount("", targetPath, "", msBind|msRemount|msRdonly, ""); err != nil { - slog.ErrorContext(ctx, "remount ro failed", "err", err) - // try to unmount in case remount failed partially - _ = unixUnmount(targetPath, 0) - return &layer, err - } - r.nydusMetaLayer.Store(snapshotID, targetPath) - return &layer, nil + targetPath := filepath.Join(r.rootDir, "store", snapshotID, target.Digest.String(), "diff") + var mountPoint string + if mountPoint, err = r.nydusFs.MountPoint(snapshotID); err == nil { + if err := os.MkdirAll(targetPath, 0755); err != nil { + slog.ErrorContext(ctx, "ensure targetPath failed", "err", err) + return &layer, err } - slog.ErrorContext(ctx, "get mount point failed", "err", err) - return &layer, err + // Perform a bind mount and then remount read-only for robustness across kernels + if err = unixMount(mountPoint, targetPath, "", msBind|msRec, ""); err != nil { + slog.ErrorContext(ctx, "bind mount failed", "err", err) + return &layer, err + } + if err = unixMount("", targetPath, "", msBind|msRemount|msRdonly, ""); err != nil { + slog.ErrorContext(ctx, "remount ro failed", "err", err) + // try to unmount in case remount failed partially + _ = unixUnmount(targetPath, 0) + return &layer, err + } + r.nydusMetaLayer.Store(snapshotID, targetPath) + return &layer, nil + } + slog.ErrorContext(ctx, "get mount point failed", "err", err) + return &layer, err } // TODO support normal image format. return &layer, nil @@ -250,7 +249,7 @@ func (r *LayerManager) Release(ctx context.Context, refspec reference.Spec, dgst i := r.refCounter[refspec.String()][dgst.String()] if i <= 0 { if v, ok := r.nydusMetaLayer.Load(snapshotID); ok { -if err := unixUnmount(v.(string), 0); err != nil { + if err := unixUnmount(v.(string), 0); err != nil { slog.ErrorContext(ctx, "umount bind nydus failed", "ref", refspec.String(), "digest", dgst.String(), "err", err) return 0, err } @@ -268,7 +267,7 @@ if err := unixUnmount(v.(string), 0); err != nil { func (r *LayerManager) ReleaseAll(ctx context.Context) { r.nydusMetaLayer.Range(func(key, value interface{}) bool { -if err := unixUnmount(value.(string), 0); err != nil { + if err := unixUnmount(value.(string), 0); err != nil { slog.WarnContext(ctx, "umount bind nydus failed", "key", key, "value", value, "err", err) } return true @@ -287,11 +286,15 @@ func (r *LayerManager) RecoverOrphanMounts(ctx context.Context) error { return err } for _, e := range ents { - if !e.IsDir() { continue } + if !e.IsDir() { + continue + } refDir := filepath.Join(storeRoot, e.Name()) layers, _ := os.ReadDir(refDir) for _, l := range layers { - if !l.IsDir() { continue } + if !l.IsDir() { + continue + } diff := filepath.Join(refDir, l.Name(), "diff") // Best-effort unmount if err := unixUnmount(diff, 0); err == nil { diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index 6a8fb36..414b679 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -20,12 +20,14 @@ type fakeFS struct { } func (f *fakeFS) UpperPath(id string) string { return filepath.Join(f.root, "snapshots", id) } -func (f *fakeFS) PrepareMetaLayer(ctx context.Context, snapshot storage.Snapshot, annotations map[string]string) error { +func (f *fakeFS) PrepareMetaLayer(_ context.Context, _ storage.Snapshot, _ map[string]string) error { return nil } -func (f *fakeFS) Mount(ctx context.Context, snapshotID string, annotations map[string]string) error { return nil } -func (f *fakeFS) WaitUntilReady(ctx context.Context, snapshotID string) error { return nil } -func (f *fakeFS) MountPoint(snapshotID string) (string, error) { return f.mp, nil } +func (f *fakeFS) Mount(_ context.Context, _ string, _ map[string]string) error { + return nil +} +func (f *fakeFS) WaitUntilReady(_ context.Context, _ string) error { return nil } +func (f *fakeFS) MountPoint(_ string) (string, error) { return f.mp, nil } func TestResolverMetaLayerBindMountAndRemountRO(t *testing.T) { ctx := context.Background() @@ -64,7 +66,10 @@ func TestResolverMetaLayerBindMountAndRemountRO(t *testing.T) { // stub unix mount origMount := unixMount defer func() { unixMount = origMount }() - type mcall struct{ src, tgt, fstype, data string; flags uintptr } + type mcall struct { + src, tgt, fstype, data string + flags uintptr + } var calls []mcall unixMount = func(source, target, fstype string, flags uintptr, data string) error { calls = append(calls, mcall{source, target, fstype, data, flags}) @@ -129,7 +134,7 @@ func TestResolverMetaLayerCreatesTargetDir(t *testing.T) { lm := &LayerManager{refPool: p, refCounter: make(map[string]map[string]int), nydusFs: &fakeFS{root: dir, mp: mountPoint}, rootDir: dir} origMount := unixMount defer func() { unixMount = origMount }() - unixMount = func(source, target, fstype string, flags uintptr, data string) error { return nil } + unixMount = func(_, _, _ string, _ uintptr, _ string) error { return nil } snapshotID := "snap-2" _, err = lm.ResolverMetaLayer(ctx, refspec, snapshotID, dgst) if err != nil { @@ -158,7 +163,7 @@ func TestReleaseDecrementsAndUnmountsAndCleansMaps(t *testing.T) { var unmounted []string origUnmount := unixUnmount defer func() { unixUnmount = origUnmount }() - unixUnmount = func(target string, flags int) error { + unixUnmount = func(target string, _ int) error { unmounted = append(unmounted, target) return nil } diff --git a/pkg/manager/mount_shim_other.go b/pkg/manager/mount_shim_other.go index 9c1885a..92cf255 100644 --- a/pkg/manager/mount_shim_other.go +++ b/pkg/manager/mount_shim_other.go @@ -15,5 +15,5 @@ type mountFunc func(source, target, fstype string, flags uintptr, data string) e type unmountFunc func(target string, flags int) error // On non-linux, provide stubs to satisfy compilation; tests override these when needed. -var unixMount mountFunc = func(source, target, fstype string, flags uintptr, data string) error { return nil } -var unixUnmount unmountFunc = func(target string, flags int) error { return nil } +var unixMount mountFunc = func(_, _, _ string, _ uintptr, _ string) error { return nil } +var unixUnmount unmountFunc = func(_ string, _ int) error { return nil } diff --git a/pkg/manager/ref_pool.go b/pkg/manager/ref_pool.go index d1d31d2..314408a 100644 --- a/pkg/manager/ref_pool.go +++ b/pkg/manager/ref_pool.go @@ -7,11 +7,11 @@ import ( "encoding/json" "fmt" "io" + "log/slog" "os" "path/filepath" "sync" "time" - "log/slog" "github.com/containerd/containerd/images" "github.com/containerd/containerd/platforms" @@ -23,7 +23,7 @@ import ( "github.com/containers/nydus-storage-plugin/pkg/cache" "github.com/containers/nydus-storage-plugin/pkg/source" - "github.com/containers/nydus-storage-plugin/pkg/utils" + manifestutil "github.com/containers/nydus-storage-plugin/pkg/utils" ) const ( @@ -245,7 +245,7 @@ func fetchManifestPlatform(ctx context.Context, fetcher remotes.Fetcher, desc oc if err != nil { return ocispec.Manifest{}, err } - if err := utils.ValidateMediaType(p, desc.MediaType); err != nil { + if err := manifestutil.ValidateMediaType(p, desc.MediaType); err != nil { return ocispec.Manifest{}, err } if err := json.Unmarshal(p, &manifest); err != nil { @@ -258,7 +258,7 @@ func fetchManifestPlatform(ctx context.Context, fetcher remotes.Fetcher, desc oc if err != nil { return ocispec.Manifest{}, err } - if err := utils.ValidateMediaType(p, desc.MediaType); err != nil { + if err := manifestutil.ValidateMediaType(p, desc.MediaType); err != nil { return ocispec.Manifest{}, err } if err = json.Unmarshal(p, &index); err != nil { diff --git a/pkg/services/keychain/dockerconfig/dockerconfig.go b/pkg/services/keychain/dockerconfig/dockerconfig.go index ef145d3..0e690ba 100644 --- a/pkg/services/keychain/dockerconfig/dockerconfig.go +++ b/pkg/services/keychain/dockerconfig/dockerconfig.go @@ -13,7 +13,7 @@ import ( // Ported from stargz-snapshotter, copyright The stargz-snapshotter Authors. // https://github.com/containerd/stargz-snapshotter/blob/923399007a8cde1ec871072ba6678b428b40b852/service/keychain/dockerconfig/dockerconfig.go func NewDockerconfigKeychain(ctx context.Context) resolver.Credential { - return func(host string, refspec reference.Spec) (string, string, error) { + return func(host string, _ reference.Spec) (string, string, error) { cf, err := config.Load("") if err != nil { slog.WarnContext(ctx, "failed to load docker config file", "err", err) diff --git a/pkg/services/keychain/podmanauth/podmanauth.go b/pkg/services/keychain/podmanauth/podmanauth.go index 49fafcf..01e15ef 100644 --- a/pkg/services/keychain/podmanauth/podmanauth.go +++ b/pkg/services/keychain/podmanauth/podmanauth.go @@ -22,7 +22,7 @@ import ( // 2) $XDG_RUNTIME_DIR/containers/auth.json // 3) $HOME/.config/containers/auth.json func NewPodmanAuthKeychain(ctx context.Context) resolver.Credential { - return func(host string, refspec reference.Spec) (string, string, error) { + return func(host string, _ reference.Spec) (string, string, error) { path, err := findAuthFile() if err != nil || path == "" { return "", "", nil @@ -107,4 +107,4 @@ func readAuthFile(path string) (authFile, error) { return authFile{}, errors.New("no auths") } return af, nil -} \ No newline at end of file +} diff --git a/pkg/services/resolver/resolver.go b/pkg/services/resolver/resolver.go index 88ab017..952287c 100644 --- a/pkg/services/resolver/resolver.go +++ b/pkg/services/resolver/resolver.go @@ -83,7 +83,7 @@ func multiCredsFuncs(ref reference.Spec, credsFuncs ...Credential) func(string) for _, f := range credsFuncs { if username, secret, err := f(host, ref); err != nil { return "", "", err - } else if !(username == "" && secret == "") { + } else if username != "" || secret != "" { return username, secret, nil } } diff --git a/pkg/utils/manifest.go b/pkg/utils/manifest.go index e2979e4..bcaf286 100644 --- a/pkg/utils/manifest.go +++ b/pkg/utils/manifest.go @@ -1,6 +1,6 @@ // Ported from stargz-snapshotter, copyright The stargz-snapshotter Authors. // https://github.com/containerd/stargz-snapshotter/blob/6fb41553e735eb6369bb3718d4b841bfacb423aa/util/containerdutil/manifest.go#L119-L158 -package utils +package manifestutil import ( "encoding/json" From e5bf0290bb4f65469c0b9d9e1c05028fa262c939 Mon Sep 17 00:00:00 2001 From: fatelei Date: Mon, 15 Dec 2025 12:20:19 +0800 Subject: [PATCH 08/23] fix: fix mount fuse --- cmd/store/main.go | 17 ++++++++++++++--- pkg/fs/fs.go | 29 ++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/cmd/store/main.go b/cmd/store/main.go index 8a46b38..0ab1a22 100644 --- a/cmd/store/main.go +++ b/cmd/store/main.go @@ -69,7 +69,7 @@ func setupSlog(level string, toStdout bool, logDir string) error { func main() { flags := command.NewFlags() - app := &cli.App{ +app := &cli.App{ Name: "crio nydus store", Usage: "crio nydus store plugin", Version: "0.0.0", @@ -77,6 +77,8 @@ func main() { &cli.StringFlag{Name: "fs-file-mode", Usage: "octal file mode for files (e.g. 0400)"}, &cli.StringFlag{Name: "fs-dir-mode", Usage: "octal dir mode (e.g. 0500)"}, &cli.StringFlag{Name: "fs-link-mode", Usage: "octal symlink mode (e.g. 0400)"}, + &cli.BoolFlag{Name: "fs-allow-other", Value: true, Usage: "enable allow_other on FUSE mount"}, + &cli.BoolFlag{Name: "fs-direct-mount", Value: false, Usage: "force direct mount (bypass fusermount)"}, ), Action: func(c *cli.Context) error { if err := setupSlog(flags.Args.LogLevel, flags.Args.LogToStdout, flags.Args.LogDir); err != nil { @@ -106,7 +108,7 @@ func main() { panic(err) } - // Parse optional FS modes +// Parse optional FS modes fileMode := fs.DefaultFileMode() dirMode := fs.DefaultDirMode() linkMode := fs.DefaultLinkMode() @@ -129,7 +131,16 @@ func main() { // Recover orphan bind mounts from previous crashes _ = layManager.RecoverOrphanMounts(c.Context) - if err := fs.Mount(c.Context, mountPoint, flags.Args.RootDir, true, layManager, fs.WithModes(fileMode, dirMode, linkMode)); err != nil { + if err := fs.Mount( + c.Context, + mountPoint, + flags.Args.RootDir, + true, + layManager, + fs.WithModes(fileMode, dirMode, linkMode), + fs.WithAllowOther(c.Bool("fs-allow-other")), + fs.WithDirectMount(c.Bool("fs-direct-mount")), + ); err != nil { slog.ErrorContext(c.Context, "failed to mount fs", "mountPoint", mountPoint, "err", err) return err } diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 3d7ae1b..1855072 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -32,6 +32,10 @@ var ( defaultDirMode uint32 = syscall.S_IFDIR | 0500 // dr-x------ defaultFileMode uint32 = 0400 // -r-------- layerFileMode uint32 = 0400 // -r-------- + + // Mount behavior toggles + defaultAllowOther bool = true + forceDirectMount bool = false ) // Helpers to expose defaults for CLI parsing @@ -52,6 +56,20 @@ func WithModes(fileMode, dirMode, linkMode uint32) MountOption { } } +// WithAllowOther toggles the allow_other mount option. +func WithAllowOther(b bool) MountOption { + return func() { + defaultAllowOther = b + } +} + +// WithDirectMount forces direct mount, bypassing fusermount helpers. +func WithDirectMount(b bool) MountOption { + return func() { + forceDirectMount = b + } +} + type releasable interface { releasable() bool } @@ -120,15 +138,20 @@ func Mount(_ context.Context, mountPoint string, _ string, debug bool, layManage NullPermissions: true, }) mountOpts := &fuse.MountOptions{ - AllowOther: true, // allow users other than root&mounter to access fs + AllowOther: defaultAllowOther, // allow users other than root&mounter to access fs FsName: "nydusstore", Debug: debug, } // Detect fusermount or fusermount3; fallback to direct mount if neither present - if hasFusermount() { + if hasFusermount() && !forceDirectMount { mountOpts.Options = []string{"suid"} // allow setuid inside container } else { - slog.Debug("fusermount/fusermount3 not installed; trying direct mount") + if !hasFusermount() { + slog.Debug("fusermount/fusermount3 not installed; trying direct mount") + } + if forceDirectMount { + slog.Debug("forcing direct mount per option") + } mountOpts.DirectMount = true } server, err := fuse.NewServer(rawFS, mountPoint, mountOpts) From 742de0e61632a6492e6dcd90f3b5fa9bdcc54480 Mon Sep 17 00:00:00 2001 From: fatelei Date: Mon, 15 Dec 2025 13:47:06 +0800 Subject: [PATCH 09/23] fix: fix log --- cmd/store/main.go | 6 ++++-- pkg/fs/fs.go | 12 ++++++++++-- pkg/fs/layer_node.go | 6 +++--- pkg/fs/ref_node.go | 2 +- pkg/fs/root_node.go | 2 +- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/cmd/store/main.go b/cmd/store/main.go index 0ab1a22..e53329a 100644 --- a/cmd/store/main.go +++ b/cmd/store/main.go @@ -78,7 +78,8 @@ app := &cli.App{ &cli.StringFlag{Name: "fs-dir-mode", Usage: "octal dir mode (e.g. 0500)"}, &cli.StringFlag{Name: "fs-link-mode", Usage: "octal symlink mode (e.g. 0400)"}, &cli.BoolFlag{Name: "fs-allow-other", Value: true, Usage: "enable allow_other on FUSE mount"}, - &cli.BoolFlag{Name: "fs-direct-mount", Value: false, Usage: "force direct mount (bypass fusermount)"}, +&cli.BoolFlag{Name: "fs-direct-mount", Value: false, Usage: "force direct mount (bypass fusermount)"}, + &cli.BoolFlag{Name: "fs-mount-suid", Value: false, Usage: "add suid to fusermount mount options"}, ), Action: func(c *cli.Context) error { if err := setupSlog(flags.Args.LogLevel, flags.Args.LogToStdout, flags.Args.LogDir); err != nil { @@ -139,7 +140,8 @@ app := &cli.App{ layManager, fs.WithModes(fileMode, dirMode, linkMode), fs.WithAllowOther(c.Bool("fs-allow-other")), - fs.WithDirectMount(c.Bool("fs-direct-mount")), +fs.WithDirectMount(c.Bool("fs-direct-mount")), + fs.WithMountSuid(c.Bool("fs-mount-suid")), ); err != nil { slog.ErrorContext(c.Context, "failed to mount fs", "mountPoint", mountPoint, "err", err) return err diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 1855072..be79429 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -36,6 +36,7 @@ var ( // Mount behavior toggles defaultAllowOther bool = true forceDirectMount bool = false + enableSuidOption bool = false ) // Helpers to expose defaults for CLI parsing @@ -70,6 +71,11 @@ func WithDirectMount(b bool) MountOption { } } +// WithMountSuid toggles "suid" mount option when using fusermount. +func WithMountSuid(b bool) MountOption { + return func() { enableSuidOption = b } +} + type releasable interface { releasable() bool } @@ -143,8 +149,10 @@ func Mount(_ context.Context, mountPoint string, _ string, debug bool, layManage Debug: debug, } // Detect fusermount or fusermount3; fallback to direct mount if neither present - if hasFusermount() && !forceDirectMount { - mountOpts.Options = []string{"suid"} // allow setuid inside container +if hasFusermount() && !forceDirectMount { + if enableSuidOption { + mountOpts.Options = append(mountOpts.Options, "suid") // optional + } } else { if !hasFusermount() { slog.Debug("fusermount/fusermount3 not installed; trying direct mount") diff --git a/pkg/fs/layer_node.go b/pkg/fs/layer_node.go index acca5df..22a0609 100644 --- a/pkg/fs/layer_node.go +++ b/pkg/fs/layer_node.go @@ -41,7 +41,7 @@ func (n *layerNode) Create(ctx context.Context, name string, _ uint32, _ uint32, // Lookup routes to the target file stored in the pool, based on the specified file name. func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fusefs.Inode, syscall.Errno) { - slog.DebugContext(ctx, "layer node lookup", "name", name) + slog.InfoContext(ctx, "layer node lookup", "name", name) switch name { case layerInfoLink: info, err := n.fs.layManager.GetLayerInfo(ctx, n.refNode.ref, n.digest) @@ -103,7 +103,7 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) // Only Nydus meta layers expose a diff directory via bind mount if !l.IsMetaLayer || l.MountFailed { - slog.DebugContext(ctx, "not a nydus meta layer; no diff provided", "digest", n.digest.String()) + slog.InfoContext(ctx, "not a nydus meta layer; no diff provided", "digest", n.digest.String()) return nil, syscall.ENOENT } @@ -131,7 +131,7 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) return cn }) case layerUseFile: - slog.DebugContext(ctx, "use file referred; returning ENOENT for reference mgmt") + slog.InfoContext(ctx, "use file referred; returning ENOENT for reference mgmt") return nil, syscall.ENOENT default: slog.WarnContext(ctx, "unknown filename", "name", name) diff --git a/pkg/fs/ref_node.go b/pkg/fs/ref_node.go index 366ddc3..602762b 100644 --- a/pkg/fs/ref_node.go +++ b/pkg/fs/ref_node.go @@ -28,7 +28,7 @@ var _ = (fusefs.NodeRmdirer)((*refNode)(nil)) func (n *refNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fusefs.Inode, syscall.Errno) { // lookup on memory nodes - slog.DebugContext(ctx, "ref node lookup", "name", name) + slog.InfoContext(ctx, "ref node lookup", "name", name) if child := n.GetChild(name); child != nil { switch tn := child.Operations().(type) { case *layerNode: diff --git a/pkg/fs/root_node.go b/pkg/fs/root_node.go index 3a13301..97bc3c5 100644 --- a/pkg/fs/root_node.go +++ b/pkg/fs/root_node.go @@ -26,7 +26,7 @@ var _ = (fusefs.NodeLookuper)((*rootNode)(nil)) // Lookup loads manifest and config of specified name (image reference) // and returns refnode of the specified name func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fusefs.Inode, syscall.Errno) { - slog.DebugContext(ctx, "root node lookup", "name", name) + slog.InfoContext(ctx, "root node lookup", "name", name) if child := n.GetChild(name); child != nil { switch tn := child.Operations().(type) { case *fusefs.MemSymlink: From be79ae504a65bc7cf31b6d203376bad4b9673e87 Mon Sep 17 00:00:00 2001 From: fatelei Date: Mon, 15 Dec 2025 14:49:07 +0800 Subject: [PATCH 10/23] fix: fix readdir --- pkg/fs/root_node.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/fs/root_node.go b/pkg/fs/root_node.go index 97bc3c5..8cbf97d 100644 --- a/pkg/fs/root_node.go +++ b/pkg/fs/root_node.go @@ -22,6 +22,7 @@ type rootNode struct { var _ = (fusefs.InodeEmbedder)((*rootNode)(nil)) var _ = (fusefs.NodeLookuper)((*rootNode)(nil)) +var _ = (fusefs.NodeReaddirer)((*rootNode)(nil)) // Lookup loads manifest and config of specified name (image reference) // and returns refnode of the specified name @@ -81,3 +82,17 @@ func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) return n.NewInode(ctx, child, sAttr) }) } + +// Readdir enumerates entries in the root directory. +// Shows the "pool" symlink and any cached image references. +func (n *rootNode) Readdir(ctx context.Context) (fusefs.DirStream, syscall.Errno) { + // Start with the pool symlink + entries := []fuse.DirEntry{ + {Name: poolLink, Mode: fuse.S_IFLNK}, + } + + // TODO: Add existing image references + // Currently, we only show the pool symlink since that's what's always available + + return fusefs.NewListDirStream(entries), 0 +} From 51f3bafb97376d6b766b9aedd6195a2bfff93b25 Mon Sep 17 00:00:00 2001 From: fatelei Date: Mon, 15 Dec 2025 15:07:47 +0800 Subject: [PATCH 11/23] fix: fix layer inspect --- pkg/manager/manager.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index e5bb9e1..3eecafa 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -150,12 +150,17 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. } // Download nydus bootstrap layer and mount it. - if _, ok := target.Annotations[label.NydusMetaLayer]; ok { + // Support both legacy and new nydus bootstrap annotations + isNydusBootstrap := target.Annotations != nil && ( + target.Annotations[label.NydusMetaLayer] == "true" || + target.Annotations["containerd.io/snapshot/nydus-bootstrap"] == "true") + + if isNydusBootstrap { target.Annotations[label.CRIImageRef] = refspec.String() target.Annotations[label.CRILayerDigest] = target.Digest.String() layer.IsMetaLayer = true - if _, ok = r.nydusMetaLayer.Load(snapshotID); ok { + if _, exists := r.nydusMetaLayer.Load(snapshotID); exists { slog.WarnContext(ctx, "nydus duplicate mount meta layer", "ref", refspec.String(), "digest", target.Digest.String()) return &layer, nil } From 1295beebdb46785dd2a26c3afafdfa9aa5fe324b Mon Sep 17 00:00:00 2001 From: fatelei Date: Mon, 15 Dec 2025 15:22:42 +0800 Subject: [PATCH 12/23] chore: add debug log --- cmd/store/main.go | 8 ++++++++ pkg/manager/manager.go | 42 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/cmd/store/main.go b/cmd/store/main.go index e53329a..f2f0b51 100644 --- a/cmd/store/main.go +++ b/cmd/store/main.go @@ -132,6 +132,14 @@ app := &cli.App{ // Recover orphan bind mounts from previous crashes _ = layManager.RecoverOrphanMounts(c.Context) + slog.InfoContext(c.Context, "Starting FUSE mount", + "mountPoint", mountPoint, + "rootDir", flags.Args.RootDir, + "fileMode", fmt.Sprintf("0%o", fileMode), + "dirMode", fmt.Sprintf("0%o", dirMode), + "linkMode", fmt.Sprintf("0%o", linkMode), + "allowOther", c.Bool("fs-allow-other")) + if err := fs.Mount( c.Context, mountPoint, diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 3eecafa..36f1b3b 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -47,6 +47,12 @@ type MountLayer struct { var ErrMountMetaLayerFailed = errors.New("mount meta layer failed") func NewLayerManager(ctx context.Context, rootDir string, hosts source.RegistryHosts, cfg *config.Config) (*LayerManager, error) { + slog.InfoContext(ctx, "NewLayerManager called", + "rootDir", rootDir, + "nydusdBinaryPath", cfg.NydusdBinaryPath, + "daemonMode", cfg.DaemonMode, + "cacheDir", cfg.CacheDir) + verifier, err := signature.NewVerifier(cfg.PublicKeyFile, cfg.ValidateSignature) if err != nil { return nil, err @@ -125,18 +131,35 @@ func (r *LayerManager) GetLayerInfo(ctx context.Context, refspec reference.Spec, } func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference.Spec, snapshotID string, digest digest.Digest) (*MountLayer, error) { + slog.InfoContext(ctx, "ResolverMetaLayer called", + "ref", refspec.String(), + "snapshotID", snapshotID, + "digest", digest.String()) + // get manifest from cache. manifest, _, err := r.refPool.loadRef(ctx, refspec) if err != nil { return nil, fmt.Errorf("failed to get manifest and config: %w", err) } + + slog.InfoContext(ctx, "manifest layers count", "count", len(manifest.Layers)) + var target ocispec.Descriptor var found bool - for _, l := range manifest.Layers { + for i, l := range manifest.Layers { + slog.InfoContext(ctx, "checking layer", + "index", i, + "digest", l.Digest.String(), + "mediaType", l.MediaType, + "annotations", l.Annotations) if l.Digest == digest { l := l found = true target = l + slog.InfoContext(ctx, "found target layer", + "digest", digest.String(), + "mediaType", l.MediaType, + "annotations", l.Annotations) break } } @@ -151,10 +174,27 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. // Download nydus bootstrap layer and mount it. // Support both legacy and new nydus bootstrap annotations + slog.InfoContext(ctx, "checking if layer is nydus bootstrap", + "digest", target.Digest.String(), + "annotations", target.Annotations) + isNydusBootstrap := target.Annotations != nil && ( target.Annotations[label.NydusMetaLayer] == "true" || target.Annotations["containerd.io/snapshot/nydus-bootstrap"] == "true") + slog.InfoContext(ctx, "nydus bootstrap check result", + "digest", target.Digest.String(), + "isBootstrap", isNydusBootstrap, + "hasAnnotations", target.Annotations != nil) + + if target.Annotations != nil { + for k, v := range target.Annotations { + slog.InfoContext(ctx, "layer annotation", + "key", k, + "value", v) + } + } + if isNydusBootstrap { target.Annotations[label.CRIImageRef] = refspec.String() target.Annotations[label.CRILayerDigest] = target.Digest.String() From d2d51de8028bb5445c4aac898a3c2ca899d5f961 Mon Sep 17 00:00:00 2001 From: fatelei Date: Mon, 15 Dec 2025 15:41:45 +0800 Subject: [PATCH 13/23] chore: debug me --- pkg/fs/root_node.go | 31 +++++++++++++++++++++++++++++++ pkg/manager/manager.go | 5 +++++ 2 files changed, 36 insertions(+) diff --git a/pkg/fs/root_node.go b/pkg/fs/root_node.go index 8cbf97d..0cb628f 100644 --- a/pkg/fs/root_node.go +++ b/pkg/fs/root_node.go @@ -23,6 +23,7 @@ var _ = (fusefs.InodeEmbedder)((*rootNode)(nil)) var _ = (fusefs.NodeLookuper)((*rootNode)(nil)) var _ = (fusefs.NodeReaddirer)((*rootNode)(nil)) +var _ = (fusefs.NodeUnlinker)((*rootNode)(nil)) // Lookup loads manifest and config of specified name (image reference) // and returns refnode of the specified name @@ -54,6 +55,20 @@ func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) sAttr.Ino = uint64(ino) return n.NewInode(ctx, cn, sAttr) }) + case "test-nydus-store-alive": + // Test file to verify Podman is accessing our FUSE mount + slog.InfoContext(ctx, "TEST: Podman accessed our test file!", "name", name) + sAttr := defaultFileAttr(uint64(len("nydus-storage-plugin-is-alive")), &out.Attr) + cn := &fusefs.MemRegularFile{ + Data: []byte("nydus-storage-plugin-is-alive"), + } + copyAttr(&cn.Attr, &out.Attr) + return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { + out.Ino = uint64(ino) + cn.Attr.Ino = uint64(ino) + sAttr.Ino = uint64(ino) + return n.NewInode(ctx, cn, sAttr) + }) } refBytes, err := base64.StdEncoding.DecodeString(name) @@ -89,6 +104,7 @@ func (n *rootNode) Readdir(ctx context.Context) (fusefs.DirStream, syscall.Errno // Start with the pool symlink entries := []fuse.DirEntry{ {Name: poolLink, Mode: fuse.S_IFLNK}, + {Name: "test-nydus-store-alive", Mode: fuse.S_IFREG}, } // TODO: Add existing image references @@ -96,3 +112,18 @@ func (n *rootNode) Readdir(ctx context.Context) (fusefs.DirStream, syscall.Errno return fusefs.NewListDirStream(entries), 0 } + +// Unlink prevents deletion of critical system entries like "pool" +func (n *rootNode) Unlink(ctx context.Context, name string) syscall.Errno { + slog.InfoContext(ctx, "root node unlink attempt", "name", name) + + // Prevent deletion of the pool symlink + if name == poolLink { + slog.WarnContext(ctx, "attempted to delete protected pool symlink", "name", name) + return syscall.EPERM // Operation not permitted + } + + // For other files, we don't support deletion + slog.InfoContext(ctx, "unlink not supported", "name", name) + return syscall.EPERM +} diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 36f1b3b..058ac02 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -323,13 +323,18 @@ func (r *LayerManager) ReleaseAll(ctx context.Context) { // left over from previous crashes. It walks /store/*/*/diff and tries to unmount. func (r *LayerManager) RecoverOrphanMounts(ctx context.Context) error { storeRoot := filepath.Join(r.rootDir, "store") + slog.InfoContext(ctx, "RecoverOrphanMounts checking store", "storeRoot", storeRoot) + ents, err := os.ReadDir(storeRoot) if err != nil { if os.IsNotExist(err) { + slog.InfoContext(ctx, "store directory does not exist, skipping recovery", "storeRoot", storeRoot) return nil } return err } + + slog.InfoContext(ctx, "found entries in store", "count", len(ents), "storeRoot", storeRoot) for _, e := range ents { if !e.IsDir() { continue From 98ad75d1690e66473623bf9a2c013151616e1c52 Mon Sep 17 00:00:00 2001 From: fatelei Date: Mon, 15 Dec 2025 16:07:15 +0800 Subject: [PATCH 14/23] chore: debug me --- pkg/fs/root_node.go | 44 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/pkg/fs/root_node.go b/pkg/fs/root_node.go index 0cb628f..c03cc34 100644 --- a/pkg/fs/root_node.go +++ b/pkg/fs/root_node.go @@ -6,6 +6,7 @@ import ( "context" "encoding/base64" "log/slog" + "strings" "syscall" "github.com/containerd/containerd/reference" @@ -13,6 +14,13 @@ import ( "github.com/hanwen/go-fuse/v2/fuse" ) +func min(a, b int) int { + if a < b { + return a + } + return b +} + // rootnode is the mountpoint node of nydus-store. type rootNode struct { fusefs.Inode @@ -71,10 +79,40 @@ func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) }) } - refBytes, err := base64.StdEncoding.DecodeString(name) + // Handle system files (starting with .) gracefully + if strings.HasPrefix(name, ".") { + slog.DebugContext(ctx, "ignoring system file", "name", name) + return nil, syscall.ENOENT // File not found (better than EINVAL) + } + + // Try multiple base64 encodings + var refBytes []byte + var err error + + // Try standard base64 first + refBytes, err = base64.StdEncoding.DecodeString(name) if err != nil { - slog.ErrorContext(ctx, "failed to decode ref base64", "name", name, "err", err) - return nil, syscall.EINVAL + // Try URL-safe base64 (without padding) + refBytes, err = base64.RawURLEncoding.DecodeString(name) + } + if err != nil { + // Try raw standard base64 (without padding) + refBytes, err = base64.RawStdEncoding.DecodeString(name) + } + + if err != nil { + slog.ErrorContext(ctx, "failed to decode base64 reference", + "name", name, + "name_len", len(name), + "err", err) + // Try to decode as much as possible for debugging + if len(name) >= 4 { + partial := name[:min(len(name), 32)] + if partialBytes, partialErr := base64.StdEncoding.DecodeString(partial + "==="); partialErr == nil { + slog.InfoContext(ctx, "partial decode result", "partial", partial, "decoded", string(partialBytes)) + } + } + return nil, syscall.ENOENT // File not found (better than EINVAL) } ref := string(refBytes) var refSpec reference.Spec From 4279737fac11e7e323d1735f932bdf9de61e4a2a Mon Sep 17 00:00:00 2001 From: fatelei Date: Mon, 15 Dec 2025 17:18:56 +0800 Subject: [PATCH 15/23] chore: debug me --- pkg/fs/blob_file.go | 71 +++++++++++++++++++++++++++++++++--------- pkg/fs/blob_node.go | 15 +++++++-- pkg/fs/layer_node.go | 2 +- pkg/fs/root_node.go | 14 --------- pkg/manager/manager.go | 3 ++ 5 files changed, 74 insertions(+), 31 deletions(-) diff --git a/pkg/fs/blob_file.go b/pkg/fs/blob_file.go index 769327d..0ba4f0a 100644 --- a/pkg/fs/blob_file.go +++ b/pkg/fs/blob_file.go @@ -2,33 +2,76 @@ package fs import ( "context" + "io" + "log/slog" + "sync" "syscall" + "github.com/containerd/containerd/remotes/docker" fusefs "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// blob file is the file handle of blob contents. +// blobFile streams a remote blob sequentially via containerd's resolver. type blobFile struct { + mu sync.Mutex + rc io.ReadCloser + size int64 + pos int64 +} + +func newBlobFile(ctx context.Context, ref string, desc ocispec.Descriptor, hostsFn func(string) ([]docker.RegistryHost, error)) (*blobFile, error) { + resolver := docker.NewResolver(docker.ResolverOptions{ + Hosts: func(host string) ([]docker.RegistryHost, error) { + return hostsFn(host) + }, + }) + fetcher, err := resolver.Fetcher(ctx, ref) + if err != nil { + return nil, err + } + r, err := fetcher.Fetch(ctx, desc) + if err != nil { + return nil, err + } + return &blobFile{rc: r, size: desc.Size, pos: 0}, nil } var _ = (fusefs.FileReader)((*blobFile)(nil)) +var _ = (fusefs.FileReleaser)((*blobFile)(nil)) +var _ = (fusefs.FileGetattrer)((*blobFile)(nil)) -func (f *blobFile) Read(_ context.Context, _ []byte, _ int64) (fuse.ReadResult, syscall.Errno) { - //s, err := f.l.ReadAt(dest, off, - // remote.WithContext(ctx), // Make cancellable - // remote.WithCacheOpts(cache.Direct()), // Do not pollute mem cache - //) - //if err != nil && err != io.EOF { - // return nil, syscall.EIO - //} - //return fuse.ReadResultData(dest[:s]), 0 - return nil, syscall.EIO +func (f *blobFile) Read(_ context.Context, dest []byte, off int64) (fuse.ReadResult, syscall.Errno) { + f.mu.Lock() + defer f.mu.Unlock() + if off != f.pos { + // Only sequential reads are supported in this initial implementation + slog.Warn("non-sequential read on blob; returning EIO", "off", off, "pos", f.pos) + return nil, syscall.EIO + } + n, err := io.ReadAtLeast(f.rc, dest, 1) + if err == io.EOF { + return fuse.ReadResultData(dest[:0]), 0 + } + if err != nil && err != io.ErrUnexpectedEOF { + return nil, syscall.EIO + } + f.pos += int64(n) + return fuse.ReadResultData(dest[:n]), 0 } -var _ = (fusefs.FileGetattrer)((*blobFile)(nil)) +func (f *blobFile) Release(_ context.Context) syscall.Errno { + f.mu.Lock() + defer f.mu.Unlock() + if f.rc != nil { + _ = f.rc.Close() + f.rc = nil + } + return 0 +} -func (f *blobFile) Getattr(_ context.Context, _ *fuse.AttrOut) syscall.Errno { - //layerToAttr(f.l, &out.Attr) +func (f *blobFile) Getattr(_ context.Context, out *fuse.AttrOut) syscall.Errno { + out.Size = uint64(f.size) return 0 } diff --git a/pkg/fs/blob_node.go b/pkg/fs/blob_node.go index 52708bc..1bc5052 100644 --- a/pkg/fs/blob_node.go +++ b/pkg/fs/blob_node.go @@ -4,6 +4,8 @@ import ( "context" "syscall" + "github.com/containerd/containerd/reference" + "github.com/containerd/containerd/remotes/docker" fusefs "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -14,12 +16,21 @@ type blobNode struct { fusefs.Inode attr fuse.Attr l *ocispec.Descriptor + fs *fs + ref reference.Spec } var _ = (fusefs.InodeEmbedder)((*blobNode)(nil)) var _ = (fusefs.NodeOpener)((*blobNode)(nil)) -func (n *blobNode) Open(_ context.Context, _ uint32) (fh fusefs.FileHandle, fuseFlags uint32, errno syscall.Errno) { - return &blobFile{}, 0, 0 +func (n *blobNode) Open(ctx context.Context, _ uint32) (fh fusefs.FileHandle, fuseFlags uint32, errno syscall.Errno) { + hostsFn := func(_ string) ([]docker.RegistryHost, error) { + return n.fs.layManager.Hosts()(n.ref) + } + bf, err := newBlobFile(ctx, n.ref.String(), *n.l, hostsFn) + if err != nil { + return nil, 0, syscall.EIO + } + return bf, 0, 0 } diff --git a/pkg/fs/layer_node.go b/pkg/fs/layer_node.go index 22a0609..148cb03 100644 --- a/pkg/fs/layer_node.go +++ b/pkg/fs/layer_node.go @@ -91,7 +91,7 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) } if name == blobLink { sAttr := layerToAttr(&l.Descriptor, &out.Attr) - cn := &blobNode{l: &l.Descriptor} + cn := &blobNode{l: &l.Descriptor, fs: n.fs, ref: n.refNode.ref} copyAttr(&cn.attr, &out.Attr) return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { out.Ino = uint64(ino) diff --git a/pkg/fs/root_node.go b/pkg/fs/root_node.go index c03cc34..a288c99 100644 --- a/pkg/fs/root_node.go +++ b/pkg/fs/root_node.go @@ -63,20 +63,6 @@ func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) sAttr.Ino = uint64(ino) return n.NewInode(ctx, cn, sAttr) }) - case "test-nydus-store-alive": - // Test file to verify Podman is accessing our FUSE mount - slog.InfoContext(ctx, "TEST: Podman accessed our test file!", "name", name) - sAttr := defaultFileAttr(uint64(len("nydus-storage-plugin-is-alive")), &out.Attr) - cn := &fusefs.MemRegularFile{ - Data: []byte("nydus-storage-plugin-is-alive"), - } - copyAttr(&cn.Attr, &out.Attr) - return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { - out.Ino = uint64(ino) - cn.Attr.Ino = uint64(ino) - sAttr.Ino = uint64(ino) - return n.NewInode(ctx, cn, sAttr) - }) } // Handle system files (starting with .) gracefully diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 058ac02..5bd3435 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -379,6 +379,9 @@ func (r *LayerManager) RefRoot() string { return r.refPool.root() } +// Hosts returns the registry hosts provider. +func (r *LayerManager) Hosts() source.RegistryHosts { return r.hosts } + func colon2dash(s string) string { return strings.ReplaceAll(s, ":", "-") } From 1269170cd755b0d142730690697bd8346899dd30 Mon Sep 17 00:00:00 2001 From: fatelei Date: Mon, 15 Dec 2025 17:49:41 +0800 Subject: [PATCH 16/23] chore: debug me --- pkg/fs/fs.go | 9 +++++---- pkg/fs/utils.go | 16 ++++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index be79429..d0ec93b 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -28,10 +28,11 @@ const ( // Configurable FS modes (defaults preserved) var ( - defaultLinkMode uint32 = syscall.S_IFLNK | 0400 // -r-------- - defaultDirMode uint32 = syscall.S_IFDIR | 0500 // dr-x------ - defaultFileMode uint32 = 0400 // -r-------- - layerFileMode uint32 = 0400 // -r-------- + // Permission bits only (no file type bits). File type is set via fuse StableAttr. + defaultLinkMode uint32 = 0400 // -r-------- (symlink perms are largely ignored by kernels) + defaultDirMode uint32 = 0500 // dr-x------ + defaultFileMode uint32 = 0400 // -r-------- + layerFileMode uint32 = 0400 // -r-------- // Mount behavior toggles defaultAllowOther bool = true diff --git a/pkg/fs/utils.go b/pkg/fs/utils.go index afe921c..316308a 100644 --- a/pkg/fs/utils.go +++ b/pkg/fs/utils.go @@ -50,11 +50,11 @@ func layerToAttr(l *ocispec.Descriptor, out *fuse.Attr) fusefs.StableAttr { out.Blocks++ } out.Nlink = 1 - out.Mode = layerFileMode + out.Mode = layerFileMode // permission bits only out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ - Mode: out.Mode, + Mode: fuse.S_IFREG, // file type here } } @@ -68,10 +68,10 @@ func defaultFileAttr(size uint64, out *fuse.Attr) fusefs.StableAttr { out.Blocks++ } out.Nlink = 1 - out.Mode = defaultFileMode + out.Mode = defaultFileMode // permission bits only out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ - Mode: out.Mode, + Mode: fuse.S_IFREG, // file type here } } @@ -79,10 +79,10 @@ func defaultFileAttr(size uint64, out *fuse.Attr) fusefs.StableAttr { // https://github.com/containerd/stargz-snapshotter/blob/efc4166e93a22804b90e27c912eff7ecc0a12dfc/store/fs.go#L557 func defaultDirAttr(out *fuse.Attr) fusefs.StableAttr { out.Size = 0 - out.Mode = defaultDirMode + out.Mode = defaultDirMode // permission bits only out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ - Mode: out.Mode, + Mode: fuse.S_IFDIR, // directory type here } } @@ -90,9 +90,9 @@ func defaultDirAttr(out *fuse.Attr) fusefs.StableAttr { // https://github.com/containerd/stargz-snapshotter/blob/efc4166e93a22804b90e27c912eff7ecc0a12dfc/store/fs.go#L575 func defaultLinkAttr(out *fuse.Attr) fusefs.StableAttr { out.Size = 0 - out.Mode = defaultLinkMode + out.Mode = defaultLinkMode // permission bits only out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ - Mode: out.Mode, + Mode: fuse.S_IFLNK, // symlink type here } } From 70f9150b428b362e62c06f6948197fe7c3cd55fb Mon Sep 17 00:00:00 2001 From: fatelei Date: Mon, 15 Dec 2025 18:03:34 +0800 Subject: [PATCH 17/23] chore: debug me --- pkg/fs/blob_file.go | 82 +++++++++++++++++++++++++++++---------------- pkg/fs/utils.go | 8 ++--- 2 files changed, 58 insertions(+), 32 deletions(-) diff --git a/pkg/fs/blob_file.go b/pkg/fs/blob_file.go index 0ba4f0a..2b80039 100644 --- a/pkg/fs/blob_file.go +++ b/pkg/fs/blob_file.go @@ -13,51 +13,77 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// blobFile streams a remote blob sequentially via containerd's resolver. +// blobFile streams a remote blob and supports random access by reopening +// the stream and skipping to the requested offset when needed. type blobFile struct { - mu sync.Mutex - rc io.ReadCloser - size int64 - pos int64 + mu sync.Mutex + rc io.ReadCloser + size int64 + pos int64 + ref string + desc ocispec.Descriptor + hostsFn func(string) ([]docker.RegistryHost, error) } -func newBlobFile(ctx context.Context, ref string, desc ocispec.Descriptor, hostsFn func(string) ([]docker.RegistryHost, error)) (*blobFile, error) { +func newBlobFile(_ context.Context, ref string, desc ocispec.Descriptor, hostsFn func(string) ([]docker.RegistryHost, error)) (*blobFile, error) { + return &blobFile{ + size: desc.Size, + pos: 0, + ref: ref, + desc: desc, + hostsFn: hostsFn, + }, nil +} + +var _ = (fusefs.FileReader)((*blobFile)(nil)) +var _ = (fusefs.FileReleaser)((*blobFile)(nil)) +var _ = (fusefs.FileGetattrer)((*blobFile)(nil)) + +// openAt (re)opens the remote stream and skips to the specified offset. +func (f *blobFile) openAt(ctx context.Context, off int64) error { + if f.rc != nil { + _ = f.rc.Close() + f.rc = nil + } resolver := docker.NewResolver(docker.ResolverOptions{ - Hosts: func(host string) ([]docker.RegistryHost, error) { - return hostsFn(host) - }, + Hosts: func(host string) ([]docker.RegistryHost, error) { return f.hostsFn(host) }, }) - fetcher, err := resolver.Fetcher(ctx, ref) + fetcher, err := resolver.Fetcher(ctx, f.ref) if err != nil { - return nil, err + return err } - r, err := fetcher.Fetch(ctx, desc) + r, err := fetcher.Fetch(ctx, f.desc) if err != nil { - return nil, err + return err } - return &blobFile{rc: r, size: desc.Size, pos: 0}, nil + // Skip to desired offset (naive sequential skip). Can be optimized with HTTP Range later. + if off > 0 { + if _, err := io.CopyN(io.Discard, r, off); err != nil { + _ = r.Close() + return err + } + } + f.rc = r + f.pos = off + return nil } -var _ = (fusefs.FileReader)((*blobFile)(nil)) -var _ = (fusefs.FileReleaser)((*blobFile)(nil)) -var _ = (fusefs.FileGetattrer)((*blobFile)(nil)) - -func (f *blobFile) Read(_ context.Context, dest []byte, off int64) (fuse.ReadResult, syscall.Errno) { +func (f *blobFile) Read(ctx context.Context, dest []byte, off int64) (fuse.ReadResult, syscall.Errno) { f.mu.Lock() defer f.mu.Unlock() - if off != f.pos { - // Only sequential reads are supported in this initial implementation - slog.Warn("non-sequential read on blob; returning EIO", "off", off, "pos", f.pos) - return nil, syscall.EIO + if f.rc == nil || off != f.pos { + if err := f.openAt(ctx, off); err != nil { + slog.Warn("blob openAt failed", "off", off, "err", err) + return nil, syscall.EIO + } } - n, err := io.ReadAtLeast(f.rc, dest, 1) - if err == io.EOF { - return fuse.ReadResultData(dest[:0]), 0 + n, err := f.rc.Read(dest) + if n > 0 { + f.pos += int64(n) } - if err != nil && err != io.ErrUnexpectedEOF { + if err != nil && err != io.EOF { return nil, syscall.EIO } - f.pos += int64(n) return fuse.ReadResultData(dest[:n]), 0 } diff --git a/pkg/fs/utils.go b/pkg/fs/utils.go index 316308a..6701edf 100644 --- a/pkg/fs/utils.go +++ b/pkg/fs/utils.go @@ -50,7 +50,7 @@ func layerToAttr(l *ocispec.Descriptor, out *fuse.Attr) fusefs.StableAttr { out.Blocks++ } out.Nlink = 1 - out.Mode = layerFileMode // permission bits only +out.Mode = fuse.S_IFREG | layerFileMode // include file type + perms out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ @@ -68,7 +68,7 @@ func defaultFileAttr(size uint64, out *fuse.Attr) fusefs.StableAttr { out.Blocks++ } out.Nlink = 1 - out.Mode = defaultFileMode // permission bits only +out.Mode = fuse.S_IFREG | defaultFileMode // include file type + perms out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ Mode: fuse.S_IFREG, // file type here @@ -79,7 +79,7 @@ func defaultFileAttr(size uint64, out *fuse.Attr) fusefs.StableAttr { // https://github.com/containerd/stargz-snapshotter/blob/efc4166e93a22804b90e27c912eff7ecc0a12dfc/store/fs.go#L557 func defaultDirAttr(out *fuse.Attr) fusefs.StableAttr { out.Size = 0 - out.Mode = defaultDirMode // permission bits only +out.Mode = fuse.S_IFDIR | defaultDirMode // include dir type + perms out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ Mode: fuse.S_IFDIR, // directory type here @@ -90,7 +90,7 @@ func defaultDirAttr(out *fuse.Attr) fusefs.StableAttr { // https://github.com/containerd/stargz-snapshotter/blob/efc4166e93a22804b90e27c912eff7ecc0a12dfc/store/fs.go#L575 func defaultLinkAttr(out *fuse.Attr) fusefs.StableAttr { out.Size = 0 - out.Mode = defaultLinkMode // permission bits only +out.Mode = fuse.S_IFLNK | defaultLinkMode // include symlink type + perms out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ Mode: fuse.S_IFLNK, // symlink type here From 0e48b71495c01d655932911c53e875b4b4d79d0d Mon Sep 17 00:00:00 2001 From: fatelei Date: Mon, 15 Dec 2025 18:10:44 +0800 Subject: [PATCH 18/23] chore: debug me --- cmd/store/main.go | 8 ++++---- pkg/fs/fs.go | 2 +- pkg/fs/utils.go | 8 ++++---- pkg/manager/manager.go | 5 ++--- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/cmd/store/main.go b/cmd/store/main.go index f2f0b51..56658eb 100644 --- a/cmd/store/main.go +++ b/cmd/store/main.go @@ -69,7 +69,7 @@ func setupSlog(level string, toStdout bool, logDir string) error { func main() { flags := command.NewFlags() -app := &cli.App{ + app := &cli.App{ Name: "crio nydus store", Usage: "crio nydus store plugin", Version: "0.0.0", @@ -78,7 +78,7 @@ app := &cli.App{ &cli.StringFlag{Name: "fs-dir-mode", Usage: "octal dir mode (e.g. 0500)"}, &cli.StringFlag{Name: "fs-link-mode", Usage: "octal symlink mode (e.g. 0400)"}, &cli.BoolFlag{Name: "fs-allow-other", Value: true, Usage: "enable allow_other on FUSE mount"}, -&cli.BoolFlag{Name: "fs-direct-mount", Value: false, Usage: "force direct mount (bypass fusermount)"}, + &cli.BoolFlag{Name: "fs-direct-mount", Value: false, Usage: "force direct mount (bypass fusermount)"}, &cli.BoolFlag{Name: "fs-mount-suid", Value: false, Usage: "add suid to fusermount mount options"}, ), Action: func(c *cli.Context) error { @@ -109,7 +109,7 @@ app := &cli.App{ panic(err) } -// Parse optional FS modes + // Parse optional FS modes fileMode := fs.DefaultFileMode() dirMode := fs.DefaultDirMode() linkMode := fs.DefaultLinkMode() @@ -148,7 +148,7 @@ app := &cli.App{ layManager, fs.WithModes(fileMode, dirMode, linkMode), fs.WithAllowOther(c.Bool("fs-allow-other")), -fs.WithDirectMount(c.Bool("fs-direct-mount")), + fs.WithDirectMount(c.Bool("fs-direct-mount")), fs.WithMountSuid(c.Bool("fs-mount-suid")), ); err != nil { slog.ErrorContext(c.Context, "failed to mount fs", "mountPoint", mountPoint, "err", err) diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index d0ec93b..0ffc08d 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -150,7 +150,7 @@ func Mount(_ context.Context, mountPoint string, _ string, debug bool, layManage Debug: debug, } // Detect fusermount or fusermount3; fallback to direct mount if neither present -if hasFusermount() && !forceDirectMount { + if hasFusermount() && !forceDirectMount { if enableSuidOption { mountOpts.Options = append(mountOpts.Options, "suid") // optional } diff --git a/pkg/fs/utils.go b/pkg/fs/utils.go index 6701edf..5ee499e 100644 --- a/pkg/fs/utils.go +++ b/pkg/fs/utils.go @@ -50,7 +50,7 @@ func layerToAttr(l *ocispec.Descriptor, out *fuse.Attr) fusefs.StableAttr { out.Blocks++ } out.Nlink = 1 -out.Mode = fuse.S_IFREG | layerFileMode // include file type + perms + out.Mode = fuse.S_IFREG | layerFileMode // include file type + perms out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ @@ -68,7 +68,7 @@ func defaultFileAttr(size uint64, out *fuse.Attr) fusefs.StableAttr { out.Blocks++ } out.Nlink = 1 -out.Mode = fuse.S_IFREG | defaultFileMode // include file type + perms + out.Mode = fuse.S_IFREG | defaultFileMode // include file type + perms out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ Mode: fuse.S_IFREG, // file type here @@ -79,7 +79,7 @@ out.Mode = fuse.S_IFREG | defaultFileMode // include file type + perm // https://github.com/containerd/stargz-snapshotter/blob/efc4166e93a22804b90e27c912eff7ecc0a12dfc/store/fs.go#L557 func defaultDirAttr(out *fuse.Attr) fusefs.StableAttr { out.Size = 0 -out.Mode = fuse.S_IFDIR | defaultDirMode // include dir type + perms + out.Mode = fuse.S_IFDIR | defaultDirMode // include dir type + perms out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ Mode: fuse.S_IFDIR, // directory type here @@ -90,7 +90,7 @@ out.Mode = fuse.S_IFDIR | defaultDirMode // include dir type + perms // https://github.com/containerd/stargz-snapshotter/blob/efc4166e93a22804b90e27c912eff7ecc0a12dfc/store/fs.go#L575 func defaultLinkAttr(out *fuse.Attr) fusefs.StableAttr { out.Size = 0 -out.Mode = fuse.S_IFLNK | defaultLinkMode // include symlink type + perms + out.Mode = fuse.S_IFLNK | defaultLinkMode // include symlink type + perms out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ Mode: fuse.S_IFLNK, // symlink type here diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 5bd3435..bf15385 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -176,10 +176,9 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. // Support both legacy and new nydus bootstrap annotations slog.InfoContext(ctx, "checking if layer is nydus bootstrap", "digest", target.Digest.String(), - "annotations", target.Annotations) + "annotations", target.Annotations) - isNydusBootstrap := target.Annotations != nil && ( - target.Annotations[label.NydusMetaLayer] == "true" || + isNydusBootstrap := target.Annotations != nil && (target.Annotations[label.NydusMetaLayer] == "true" || target.Annotations["containerd.io/snapshot/nydus-bootstrap"] == "true") slog.InfoContext(ctx, "nydus bootstrap check result", From 79116e1491a968b19257347aa8c0ca8db9e8c183 Mon Sep 17 00:00:00 2001 From: fatelei Date: Mon, 15 Dec 2025 18:17:21 +0800 Subject: [PATCH 19/23] chore: debug me --- cmd/store/main.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cmd/store/main.go b/cmd/store/main.go index 56658eb..0a7dc70 100644 --- a/cmd/store/main.go +++ b/cmd/store/main.go @@ -8,6 +8,8 @@ import ( "path/filepath" "syscall" + "golang.org/x/sys/unix" + "github.com/containerd/nydus-snapshotter/cmd/containerd-nydus-grpc/pkg/command" "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/pkg/errdefs" @@ -155,10 +157,15 @@ func main() { return err } defer func() { + // Best-effort: release bind mounts first layManager.ReleaseAll(c.Context) - err := syscall.Unmount(mountPoint, 0) - if err != nil { - slog.ErrorContext(c.Context, "unmount failed", "err", err) + // Try a normal unmount of the FUSE mountpoint + if err := syscall.Unmount(mountPoint, 0); err != nil { + slog.WarnContext(c.Context, "unmount busy; retry with MNT_DETACH", "err", err) + // Retry with lazy unmount to avoid EBUSY due to lingering FDs + if derr := unix.Unmount(mountPoint, unix.MNT_DETACH); derr != nil { + slog.ErrorContext(c.Context, "lazy unmount failed", "err", derr) + } } slog.InfoContext(c.Context, "Exiting") }() From f698e4f1e88ad4a487268780aeddd1af00878894 Mon Sep 17 00:00:00 2001 From: fatelei Date: Mon, 15 Dec 2025 19:55:03 +0800 Subject: [PATCH 20/23] chore: debug me --- cmd/store/main.go | 16 ++++++++++------ pkg/fs/fs.go | 5 +++-- pkg/manager/mount_shim_other.go | 13 ++++++++++--- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/cmd/store/main.go b/cmd/store/main.go index 0a7dc70..b13a94d 100644 --- a/cmd/store/main.go +++ b/cmd/store/main.go @@ -6,10 +6,9 @@ import ( "os" "os/signal" "path/filepath" + "runtime" "syscall" - "golang.org/x/sys/unix" - "github.com/containerd/nydus-snapshotter/cmd/containerd-nydus-grpc/pkg/command" "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/pkg/errdefs" @@ -83,11 +82,17 @@ func main() { &cli.BoolFlag{Name: "fs-direct-mount", Value: false, Usage: "force direct mount (bypass fusermount)"}, &cli.BoolFlag{Name: "fs-mount-suid", Value: false, Usage: "add suid to fusermount mount options"}, ), - Action: func(c *cli.Context) error { +Action: func(c *cli.Context) error { if err := setupSlog(flags.Args.LogLevel, flags.Args.LogToStdout, flags.Args.LogDir); err != nil { return errors.Wrap(err, "failed to prepare logger") } + // Fail fast on unsupported platforms to avoid false-positive "mounted" states. + if runtime.GOOS != "linux" { + slog.ErrorContext(c.Context, "nydus-store requires Linux (FUSE) to mount; current OS unsupported", "GOOS", runtime.GOOS) + return errors.New("platform not supported: requires linux with FUSE") + } + var cfg config.Config if err := command.Validate(flags.Args, &cfg); err != nil { return errors.Wrap(err, "invalid argument") @@ -161,9 +166,8 @@ func main() { layManager.ReleaseAll(c.Context) // Try a normal unmount of the FUSE mountpoint if err := syscall.Unmount(mountPoint, 0); err != nil { - slog.WarnContext(c.Context, "unmount busy; retry with MNT_DETACH", "err", err) - // Retry with lazy unmount to avoid EBUSY due to lingering FDs - if derr := unix.Unmount(mountPoint, unix.MNT_DETACH); derr != nil { + slog.WarnContext(c.Context, "unmount busy; retry lazy unmount", "err", err) + if derr := lazyUnmount(mountPoint); derr != nil { slog.ErrorContext(c.Context, "lazy unmount failed", "err", derr) } } diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 0ffc08d..46bb2f4 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -154,12 +154,13 @@ func Mount(_ context.Context, mountPoint string, _ string, debug bool, layManage if enableSuidOption { mountOpts.Options = append(mountOpts.Options, "suid") // optional } + slog.Info("using fusermount helper", "allowOther", mountOpts.AllowOther) } else { if !hasFusermount() { - slog.Debug("fusermount/fusermount3 not installed; trying direct mount") + slog.Info("fusermount/fusermount3 not installed; trying direct mount") } if forceDirectMount { - slog.Debug("forcing direct mount per option") + slog.Info("forcing direct mount per option") } mountOpts.DirectMount = true } diff --git a/pkg/manager/mount_shim_other.go b/pkg/manager/mount_shim_other.go index 92cf255..0d6cff3 100644 --- a/pkg/manager/mount_shim_other.go +++ b/pkg/manager/mount_shim_other.go @@ -2,6 +2,8 @@ package manager +import "errors" + const ( // Linux mount flags used for tests; values chosen to match linux for assertions. msBind = 0x1000 @@ -14,6 +16,11 @@ type mountFunc func(source, target, fstype string, flags uintptr, data string) e type unmountFunc func(target string, flags int) error -// On non-linux, provide stubs to satisfy compilation; tests override these when needed. -var unixMount mountFunc = func(_, _, _ string, _ uintptr, _ string) error { return nil } -var unixUnmount unmountFunc = func(_ string, _ int) error { return nil } +// On non-linux, return explicit errors by default so production runs fail fast. +// Unit tests replace these vars to stub platform-specific behavior. +var unixMount mountFunc = func(_, _, _ string, _ uintptr, _ string) error { + return errors.New("mount is not supported on non-linux platforms") +} +var unixUnmount unmountFunc = func(_ string, _ int) error { + return errors.New("unmount is not supported on non-linux platforms") +} From 24d6cf3ac2996d5103de72dec2be8a067a479c23 Mon Sep 17 00:00:00 2001 From: fatelei Date: Tue, 16 Dec 2025 09:26:27 +0800 Subject: [PATCH 21/23] chore: debug me --- pkg/fs/blob_file.go | 9 ++++ pkg/fs/blob_node.go | 3 ++ pkg/fs/blob_node_test.go | 11 ++-- pkg/fs/utils_test.go | 18 +++---- pkg/manager/manager.go | 102 +++++++++++++++++++++--------------- pkg/manager/manager_test.go | 4 +- 6 files changed, 87 insertions(+), 60 deletions(-) diff --git a/pkg/fs/blob_file.go b/pkg/fs/blob_file.go index 2b80039..a042961 100644 --- a/pkg/fs/blob_file.go +++ b/pkg/fs/blob_file.go @@ -71,12 +71,21 @@ func (f *blobFile) openAt(ctx context.Context, off int64) error { func (f *blobFile) Read(ctx context.Context, dest []byte, off int64) (fuse.ReadResult, syscall.Errno) { f.mu.Lock() defer f.mu.Unlock() + + // Guard against invalid handles (primarily for tests) and avoid panics. + if f.hostsFn == nil || f.ref == "" { + return nil, syscall.EIO + } + if f.rc == nil || off != f.pos { if err := f.openAt(ctx, off); err != nil { slog.Warn("blob openAt failed", "off", off, "err", err) return nil, syscall.EIO } } + if len(dest) == 0 { + return fuse.ReadResultData(nil), 0 + } n, err := f.rc.Read(dest) if n > 0 { f.pos += int64(n) diff --git a/pkg/fs/blob_node.go b/pkg/fs/blob_node.go index 1bc5052..49a4efe 100644 --- a/pkg/fs/blob_node.go +++ b/pkg/fs/blob_node.go @@ -25,6 +25,9 @@ var _ = (fusefs.InodeEmbedder)((*blobNode)(nil)) var _ = (fusefs.NodeOpener)((*blobNode)(nil)) func (n *blobNode) Open(ctx context.Context, _ uint32) (fh fusefs.FileHandle, fuseFlags uint32, errno syscall.Errno) { + if n == nil || n.fs == nil || n.fs.layManager == nil || n.l == nil { + return nil, 0, syscall.EIO + } hostsFn := func(_ string) ([]docker.RegistryHost, error) { return n.fs.layManager.Hosts()(n.ref) } diff --git a/pkg/fs/blob_node_test.go b/pkg/fs/blob_node_test.go index 3a7eb93..60f8ccb 100644 --- a/pkg/fs/blob_node_test.go +++ b/pkg/fs/blob_node_test.go @@ -5,13 +5,10 @@ import ( "testing" ) -func TestBlobNodeOpenReturnsBlobFile(t *testing.T) { +func TestBlobNodeOpenWithoutInitReturnsError(t *testing.T) { n := &blobNode{} - fh, _, eno := n.Open(context.Background(), 0) - if eno != 0 { - t.Fatalf("Open errno=%d", eno) - } - if _, ok := fh.(*blobFile); !ok { - t.Fatalf("expected *blobFile, got %T", fh) + _, _, eno := n.Open(context.Background(), 0) + if eno == 0 { + t.Fatalf("expected non-zero errno for uninitialized blobNode") } } diff --git a/pkg/fs/utils_test.go b/pkg/fs/utils_test.go index 512fa0d..16668b4 100644 --- a/pkg/fs/utils_test.go +++ b/pkg/fs/utils_test.go @@ -23,26 +23,26 @@ func TestUtilsCopyAttr(t *testing.T) { func TestUtilsLayerToAttr(t *testing.T) { var out fuse.Attr s := layerToAttr(&ocispec.Descriptor{Size: 8192}, &out) - if out.Mode != layerFileMode { + if out.Mode != (fuse.S_IFREG | layerFileMode) { t.Fatalf("unexpected mode: %o", out.Mode) } - if s.Mode != out.Mode { - t.Fatalf("stable mode mismatch") + if s.Mode != fuse.S_IFREG { + t.Fatalf("stable mode mismatch: got %o", s.Mode) } } func TestUtilsDefaultFileDirLinkAttr(t *testing.T) { var out fuse.Attr s1 := defaultFileAttr(100, &out) - if out.Mode != defaultFileMode || s1.Mode != out.Mode { - t.Fatalf("defaultFileAttr unexpected") + if out.Mode != (fuse.S_IFREG | defaultFileMode) || s1.Mode != fuse.S_IFREG { + t.Fatalf("defaultFileAttr unexpected: out.Mode=%o stable.Mode=%o", out.Mode, s1.Mode) } s2 := defaultDirAttr(&out) - if out.Mode != defaultDirMode || s2.Mode != out.Mode { - t.Fatalf("defaultDirAttr unexpected") + if out.Mode != (fuse.S_IFDIR | defaultDirMode) || s2.Mode != fuse.S_IFDIR { + t.Fatalf("defaultDirAttr unexpected: out.Mode=%o stable.Mode=%o", out.Mode, s2.Mode) } s3 := defaultLinkAttr(&out) - if out.Mode != defaultLinkMode || s3.Mode != out.Mode { - t.Fatalf("defaultLinkAttr unexpected") + if out.Mode != (fuse.S_IFLNK | defaultLinkMode) || s3.Mode != fuse.S_IFLNK { + t.Fatalf("defaultLinkAttr unexpected: out.Mode=%o stable.Mode=%o", out.Mode, s3.Mode) } } diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index bf15385..ba26b31 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -114,10 +114,15 @@ type LayerManager struct { refPool *refPool hosts source.RegistryHosts - refCounter map[string]map[string]int - rootDir string + refCounter map[string]map[string]int + rootDir string + + // mountedSnapshots tracks snapshot IDs that already have an active nydusd mount. + mountedSnapshots sync.Map + // nydusMetaLayer tracks per-layer bind mounts (key: snapshotID+":"+layerDigest -> targetPath). nydusMetaLayer sync.Map - nydusFs fsDriver + + nydusFs fsDriver mu sync.Mutex } @@ -172,18 +177,29 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. IsMetaLayer: false, } - // Download nydus bootstrap layer and mount it. - // Support both legacy and new nydus bootstrap annotations - slog.InfoContext(ctx, "checking if layer is nydus bootstrap", + // Download nydus bootstrap/blob layer and mount it. + // NOTE: containers/image/nydus may set these annotations to non-"true" values; presence is what matters. + slog.InfoContext(ctx, "checking if layer is nydus layer", "digest", target.Digest.String(), "annotations", target.Annotations) - isNydusBootstrap := target.Annotations != nil && (target.Annotations[label.NydusMetaLayer] == "true" || - target.Annotations["containerd.io/snapshot/nydus-bootstrap"] == "true") + isNydusLayer := false + if target.Annotations != nil { + if _, ok := target.Annotations[label.NydusMetaLayer]; ok { + isNydusLayer = true + } + if _, ok := target.Annotations[label.NydusDataLayer]; ok { + isNydusLayer = true + } + // Legacy compatibility (same key as label.NydusMetaLayer in some older builds) + if _, ok := target.Annotations["containerd.io/snapshot/nydus-bootstrap"]; ok { + isNydusLayer = true + } + } - slog.InfoContext(ctx, "nydus bootstrap check result", + slog.InfoContext(ctx, "nydus layer check result", "digest", target.Digest.String(), - "isBootstrap", isNydusBootstrap, + "isNydusLayer", isNydusLayer, "hasAnnotations", target.Annotations != nil) if target.Annotations != nil { @@ -194,13 +210,14 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. } } - if isNydusBootstrap { + if isNydusLayer { target.Annotations[label.CRIImageRef] = refspec.String() target.Annotations[label.CRILayerDigest] = target.Digest.String() layer.IsMetaLayer = true - if _, exists := r.nydusMetaLayer.Load(snapshotID); exists { - slog.WarnContext(ctx, "nydus duplicate mount meta layer", "ref", refspec.String(), "digest", target.Digest.String()) + bindKey := snapshotID + ":" + target.Digest.String() + if _, exists := r.nydusMetaLayer.Load(bindKey); exists { + slog.DebugContext(ctx, "nydus duplicate bind mount", "ref", refspec.String(), "digest", target.Digest.String()) return &layer, nil } @@ -212,39 +229,39 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. } } - // Download nydus bootstrap layer to disk. - err = r.nydusFs.PrepareMetaLayer(ctx, storage.Snapshot{ID: snapshotID}, target.Annotations) - if err != nil && !strings.Contains(err.Error(), "file exists") { - slog.ErrorContext(ctx, "download snapshot files failed", "err", err) - return &layer, err - } + // Ensure the nydusd mount exists once per snapshotID. + if _, mounted := r.mountedSnapshots.Load(snapshotID); !mounted { + // Download nydus bootstrap layer to disk. + err = r.nydusFs.PrepareMetaLayer(ctx, storage.Snapshot{ID: snapshotID}, target.Annotations) + if err != nil && !strings.Contains(err.Error(), "file exists") { + slog.ErrorContext(ctx, "download snapshot files failed", "err", err) + return &layer, err + } - nydusMsgChannel := make(chan nydusMessage) + nydusMsgChannel := make(chan nydusMessage) - go func() { - slog.DebugContext(ctx, "nydus mount meta layer", "ref", refspec.String(), "digest", target.Digest.String()) - err = r.nydusFs.Mount(ctx, snapshotID, target.Annotations) - if err != nil { - slog.ErrorContext(ctx, "nydus mount failed", "err", err) - nydusMsgChannel <- nydusMessage{ - Err: err, + go func() { + slog.DebugContext(ctx, "nydus mount snapshot", "ref", refspec.String(), "digest", target.Digest.String(), "snapshotID", snapshotID) + err = r.nydusFs.Mount(ctx, snapshotID, target.Annotations) + if err != nil { + slog.ErrorContext(ctx, "nydus mount failed", "err", err) + nydusMsgChannel <- nydusMessage{Err: err} + return } - return - } + nydusMsgChannel <- nydusMessage{Err: nil} + }() - nydusMsgChannel <- nydusMessage{ - Err: nil, + event := <-nydusMsgChannel + if event.Err != nil { + return &layer, event.Err } - }() - event := <-nydusMsgChannel - if event.Err != nil { - return &layer, event.Err - } + err = r.nydusFs.WaitUntilReady(ctx, snapshotID) + if err != nil { + return &layer, ErrMountMetaLayerFailed + } - err = r.nydusFs.WaitUntilReady(ctx, snapshotID) - if err != nil { - return &layer, ErrMountMetaLayerFailed + r.mountedSnapshots.Store(snapshotID, true) } // Link nydusd mount dir to ///diff @@ -266,7 +283,7 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. _ = unixUnmount(targetPath, 0) return &layer, err } - r.nydusMetaLayer.Store(snapshotID, targetPath) + r.nydusMetaLayer.Store(bindKey, targetPath) return &layer, nil } slog.ErrorContext(ctx, "get mount point failed", "err", err) @@ -292,12 +309,13 @@ func (r *LayerManager) Release(ctx context.Context, refspec reference.Spec, dgst r.refCounter[refspec.String()][dgst.String()]-- i := r.refCounter[refspec.String()][dgst.String()] if i <= 0 { - if v, ok := r.nydusMetaLayer.Load(snapshotID); ok { + bindKey := snapshotID + ":" + dgst.String() + if v, ok := r.nydusMetaLayer.Load(bindKey); ok { if err := unixUnmount(v.(string), 0); err != nil { slog.ErrorContext(ctx, "umount bind nydus failed", "ref", refspec.String(), "digest", dgst.String(), "err", err) return 0, err } - r.nydusMetaLayer.Delete(snapshotID) + r.nydusMetaLayer.Delete(bindKey) } // No reference to this layer. release it. delete(r.refCounter[refspec.String()], dgst.String()) diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go index 414b679..d3483ba 100644 --- a/pkg/manager/manager_test.go +++ b/pkg/manager/manager_test.go @@ -157,7 +157,7 @@ func TestReleaseDecrementsAndUnmountsAndCleansMaps(t *testing.T) { snapshotID := "snap-1" // prepare counters lm.refCounter[refspec.String()] = map[string]int{dgst.String(): 1} - lm.nydusMetaLayer.Store(snapshotID, "/fake/target") + lm.nydusMetaLayer.Store(snapshotID+":"+dgst.String(), "/fake/target") lm.refPool.refcounter[refspec.String()] = &releaser{count: 1, release: func() {}} var unmounted []string @@ -184,7 +184,7 @@ func TestReleaseDecrementsAndUnmountsAndCleansMaps(t *testing.T) { if _, ok := lm.refCounter[refspec.String()]; ok { t.Errorf("ref entry not removed from refCounter") } - if _, ok := lm.nydusMetaLayer.Load(snapshotID); ok { + if _, ok := lm.nydusMetaLayer.Load(snapshotID + ":" + dgst.String()); ok { t.Errorf("nydusMetaLayer entry not deleted") } } From 81cd4af7fa6d4cdd7d3bcf09321204c0f4a3b928 Mon Sep 17 00:00:00 2001 From: fatelei Date: Tue, 16 Dec 2025 09:31:23 +0800 Subject: [PATCH 22/23] chore: add missing file --- WARP.md | 84 +++++++++++++++++++++++++++++++++++++++ cmd/store/umount_linux.go | 10 +++++ cmd/store/umount_other.go | 6 +++ 3 files changed, 100 insertions(+) create mode 100644 WARP.md create mode 100644 cmd/store/umount_linux.go create mode 100644 cmd/store/umount_other.go diff --git a/WARP.md b/WARP.md new file mode 100644 index 0000000..614cffd --- /dev/null +++ b/WARP.md @@ -0,0 +1,84 @@ +# WARP.md + +This file provides guidance to WARP (warp.dev) when working with code in this repository. + +## Common commands + +- Build (Go 1.25 pinned via `go.mod` and CI): + - Default (linux/amd64): `make build` + - Cross-compile examples: `GOOS=linux GOARCH=arm64 make build` + - Output: `bin/nydus-store` +- Lint (golangci-lint uses .golangci.yml): + - `make check` + - If missing locally: `curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b "$(go env GOPATH)"/bin v2.5.0` +- Unit tests: + - All: `go test ./...` + - Package: `go test ./pkg/fs -v` + - Single test: `go test ./pkg/manager -run '^TestName$' -v` +- Integration smoke (Linux with Podman and root privileges): + - Install nydus (version used in CI): + - `NYDUS_VERSION=v2.1.6` + - `wget https://github.com/dragonflyoss/image-service/releases/download/$NYDUS_VERSION/nydus-static-$NYDUS_VERSION-linux-amd64.tgz` + - `sudo tar xzvf nydus-static-$NYDUS_VERSION-linux-amd64.tgz --wildcards --strip-components=1 -C /usr/bin/ nydus-static/*` + - Configure storage and nydusd: + - `sudo mkdir -p /var/lib/nydus-store` + - `sudo cp misc/nydus-config.json /etc/nydusd-config.json` + - `sudo cp misc/storage.conf /etc/containers/storage.conf` + - Run store and verify: + - `nohup sudo bin/nydus-store --log-to-stdout --log-level info --config-path /etc/nydusd-config.json --root /var/lib/nydus-store &` + - `sudo podman run -it ghcr.io/dragonflyoss/image-service/nginx:nydus-latest echo hello word` + +## Running the plugin locally + +- Requires Linux kernel with FUSE, Podman/CRI-O using containers/storage, and `nydusd`/`nydus-image` installed in PATH. +- Typical launch: + - `sudo bin/nydus-store --log-to-stdout --log-level info --config-path /etc/nydusd-config.json --root /var/lib/nydus-store` +- Optional file mode overrides (octal without leading 0o): + - `--fs-file-mode 0400 --fs-dir-mode 0500 --fs-link-mode 0400` +- Credentials resolution obeys (in order): + - Docker config.json + - Podman-compatible `auth.json`: `REGISTRY_AUTH_FILE`, `$XDG_RUNTIME_DIR/containers/auth.json`, `$HOME/.config/containers/auth.json` + +## High-level architecture + +- Entry point (`cmd/store/main.go`) + - Uses flags from `containerd/nydus-snapshotter` to parse config (`--root`, `--config-path`, logging), sets up `slog`. + - Builds a registry resolver with credentials from both Docker config and Podman `auth.json`. + - Initializes the `LayerManager`, mounts a FUSE filesystem under `/store`, and blocks until SIGINT. On exit, releases mounts. +- Resolver & Keychains (`pkg/services/...`) + - `resolver.RegistryHostsFromConfig` constructs `docker.RegistryHost` with retryable HTTP client and request timeouts. + - Credential sources: + - Docker: `pkg/services/keychain/dockerconfig` (supports identity token, user/pass; Docker Hub host normalized). + - Podman: `pkg/services/keychain/podmanauth` (searches `REGISTRY_AUTH_FILE`, XDG runtime, then `$HOME/.config/containers/auth.json`; supports identity token and base64 `auth`). +- Layer management (`pkg/manager`) + - Verifies signatures (configurable public key, optional validation). + - Spawns and tracks `nydusd` processes via snapshotter’s process manager; stores state in an embedded DB under ``. + - Resolves image manifests/configs, identifies Nydus meta layers, downloads bootstrap, mounts via Nydus FS, waits for readiness. + - Exposes mounted content by bind-mounting Nydus mountpoints to `/store///diff` (read-only), reference-counted per layer. + - Crash recovery: on startup, attempts to unmount any orphaned bind mounts found under `/store/*/*/diff`. + - OS-specific shims: Linux-specific mount helpers with `mount_shim_linux.go`, safe fallbacks in `mount_shim_other.go` for non-Linux builds. +- FUSE filesystem (`pkg/fs`) + - go-fuse v2 based; presents a structured view with directories and symlinks: `pool`, `diff`, `blob`, `info`, and `use` markers. + - Default permission modes are intentionally restrictive; can be overridden at mount time via `WithModes` (wired to CLI flags). + - Detects `fusermount`/`fusermount3`; if absent, attempts direct mount. Waits for server mount completion before returning. +- Integration with containers/storage + - `misc/storage.conf` declares `additionallayerstores = [ "/var/lib/nydus-store/store:ref" ]` to register the plugin’s store. + - Podman/CRI-O can then lazy-mount Nydus layers referenced by images. + +## Project rules for Warp agents + +- Use Go 1.25 toolchain. +- Maintain compatibility with Podman `auth.json` discovery and precedence; do not regress Docker config support. +- Prefer FUSE3 (`fusermount3`) when available; fallback paths must remain functional. +- Keep FS access modes configurable via CLI flags and plumbed through `pkg/fs`. +- Preserve and improve crash/unmount recovery semantics in `LayerManager` (e.g., `RecoverOrphanMounts`, `ReleaseAll`). + +## Notable files + +- `Makefile` — build and lint targets (`build`, `check`). +- `.golangci.yml` — enabled linters/formatters. +- `misc/nydus-config.json`, `misc/storage.conf` — sample runtime configs. +- `cmd/store/main.go` — CLI entry. +- `pkg/manager/*` — layer lifecycle, mounting, recovery. +- `pkg/fs/*` — FUSE filesystem and wiring. +- `pkg/services/{resolver,keychain}/` — registry access and auth. diff --git a/cmd/store/umount_linux.go b/cmd/store/umount_linux.go new file mode 100644 index 0000000..f548335 --- /dev/null +++ b/cmd/store/umount_linux.go @@ -0,0 +1,10 @@ +//go:build linux + +package main + +import "golang.org/x/sys/unix" + +// lazyUnmount performs a lazy unmount (MNT_DETACH) on Linux. +func lazyUnmount(target string) error { + return unix.Unmount(target, unix.MNT_DETACH) +} \ No newline at end of file diff --git a/cmd/store/umount_other.go b/cmd/store/umount_other.go new file mode 100644 index 0000000..75f20ae --- /dev/null +++ b/cmd/store/umount_other.go @@ -0,0 +1,6 @@ +//go:build !linux + +package main + +// lazyUnmount is a no-op fallback on non-Linux platforms. +func lazyUnmount(_ string) error { return nil } \ No newline at end of file From 3d8580d0579a9bad51dceef6bd6bb6a63173428a Mon Sep 17 00:00:00 2001 From: fatelei Date: Tue, 16 Dec 2025 09:43:53 +0800 Subject: [PATCH 23/23] chore: debug me --- pkg/fs/layer_node.go | 6 ++++-- pkg/manager/manager.go | 25 +++++++++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/pkg/fs/layer_node.go b/pkg/fs/layer_node.go index 148cb03..431ddcf 100644 --- a/pkg/fs/layer_node.go +++ b/pkg/fs/layer_node.go @@ -101,9 +101,11 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) }) } - // Only Nydus meta layers expose a diff directory via bind mount + // Only Nydus layers expose a diff directory. + // - bootstrap layer: diff is backed by a bind mount to the nydusd mountpoint + // - data blob layers: diff is an intentionally empty directory (to satisfy Podman additional layer store expectations) if !l.IsMetaLayer || l.MountFailed { - slog.InfoContext(ctx, "not a nydus meta layer; no diff provided", "digest", n.digest.String()) + slog.InfoContext(ctx, "not a nydus layer; no diff provided", "digest", n.digest.String()) return nil, syscall.ENOENT } diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index ba26b31..e3228c1 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -184,22 +184,29 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. "annotations", target.Annotations) isNydusLayer := false + isNydusBootstrap := false + isNydusBlob := false if target.Annotations != nil { if _, ok := target.Annotations[label.NydusMetaLayer]; ok { isNydusLayer = true - } - if _, ok := target.Annotations[label.NydusDataLayer]; ok { - isNydusLayer = true + isNydusBootstrap = true } // Legacy compatibility (same key as label.NydusMetaLayer in some older builds) if _, ok := target.Annotations["containerd.io/snapshot/nydus-bootstrap"]; ok { isNydusLayer = true + isNydusBootstrap = true + } + if _, ok := target.Annotations[label.NydusDataLayer]; ok { + isNydusLayer = true + isNydusBlob = true } } slog.InfoContext(ctx, "nydus layer check result", "digest", target.Digest.String(), "isNydusLayer", isNydusLayer, + "isNydusBootstrap", isNydusBootstrap, + "isNydusBlob", isNydusBlob, "hasAnnotations", target.Annotations != nil) if target.Annotations != nil { @@ -215,6 +222,16 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. target.Annotations[label.CRILayerDigest] = target.Digest.String() layer.IsMetaLayer = true + // For nydus data blob layers, we intentionally DO NOT provide a bind-mounted diff. + // Podman still requires the diff entry to exist, so the FUSE layer will serve an empty directory. + if isNydusBlob && !isNydusBootstrap { + slog.InfoContext(ctx, "nydus data blob layer detected; serving empty diff directory", + "digest", target.Digest.String(), + "snapshotID", snapshotID) + return &layer, nil + } + + // Bootstrap layer: ensure nydusd mount exists and bind-mount its mountpoint to diff. bindKey := snapshotID + ":" + target.Digest.String() if _, exists := r.nydusMetaLayer.Load(bindKey); exists { slog.DebugContext(ctx, "nydus duplicate bind mount", "ref", refspec.String(), "digest", target.Digest.String()) @@ -231,7 +248,7 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. // Ensure the nydusd mount exists once per snapshotID. if _, mounted := r.mountedSnapshots.Load(snapshotID); !mounted { - // Download nydus bootstrap layer to disk. + // Download nydus bootstrap to disk. err = r.nydusFs.PrepareMetaLayer(ctx, storage.Snapshot{ID: snapshotID}, target.Annotations) if err != nil && !strings.Contains(err.Error(), "file exists") { slog.ErrorContext(ctx, "download snapshot files failed", "err", err)