diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 923851f..090dca4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: '1.18' + go-version: "1.25" - name: Login ghcr uses: docker/login-action@v3 with: @@ -26,7 +26,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build run: | - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.48.0 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0 export PATH=$PATH:$(go env GOPATH)/bin make check make build diff --git a/.golangci.yml b/.golangci.yml index 19981df..93ca81a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,22 +1,29 @@ -# https://golangci-lint.run/usage/configuration#config-file - +version: "2" linters: enable: - - structcheck - - varcheck - - staticcheck - - unconvert - - gofmt - - goimports - - revive - - ineffassign - - vet - - unused - misspell + - revive + - unconvert disable: - errcheck - -run: - deadline: 4m - skip-dirs: - - misc + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - gofmt + - goimports + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/WARP.md b/WARP.md new file mode 100644 index 0000000..614cffd --- /dev/null +++ b/WARP.md @@ -0,0 +1,84 @@ +# WARP.md + +This file provides guidance to WARP (warp.dev) when working with code in this repository. + +## Common commands + +- Build (Go 1.25 pinned via `go.mod` and CI): + - Default (linux/amd64): `make build` + - Cross-compile examples: `GOOS=linux GOARCH=arm64 make build` + - Output: `bin/nydus-store` +- Lint (golangci-lint uses .golangci.yml): + - `make check` + - If missing locally: `curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b "$(go env GOPATH)"/bin v2.5.0` +- Unit tests: + - All: `go test ./...` + - Package: `go test ./pkg/fs -v` + - Single test: `go test ./pkg/manager -run '^TestName$' -v` +- Integration smoke (Linux with Podman and root privileges): + - Install nydus (version used in CI): + - `NYDUS_VERSION=v2.1.6` + - `wget https://github.com/dragonflyoss/image-service/releases/download/$NYDUS_VERSION/nydus-static-$NYDUS_VERSION-linux-amd64.tgz` + - `sudo tar xzvf nydus-static-$NYDUS_VERSION-linux-amd64.tgz --wildcards --strip-components=1 -C /usr/bin/ nydus-static/*` + - Configure storage and nydusd: + - `sudo mkdir -p /var/lib/nydus-store` + - `sudo cp misc/nydus-config.json /etc/nydusd-config.json` + - `sudo cp misc/storage.conf /etc/containers/storage.conf` + - Run store and verify: + - `nohup sudo bin/nydus-store --log-to-stdout --log-level info --config-path /etc/nydusd-config.json --root /var/lib/nydus-store &` + - `sudo podman run -it ghcr.io/dragonflyoss/image-service/nginx:nydus-latest echo hello word` + +## Running the plugin locally + +- Requires Linux kernel with FUSE, Podman/CRI-O using containers/storage, and `nydusd`/`nydus-image` installed in PATH. +- Typical launch: + - `sudo bin/nydus-store --log-to-stdout --log-level info --config-path /etc/nydusd-config.json --root /var/lib/nydus-store` +- Optional file mode overrides (octal without leading 0o): + - `--fs-file-mode 0400 --fs-dir-mode 0500 --fs-link-mode 0400` +- Credentials resolution obeys (in order): + - Docker config.json + - Podman-compatible `auth.json`: `REGISTRY_AUTH_FILE`, `$XDG_RUNTIME_DIR/containers/auth.json`, `$HOME/.config/containers/auth.json` + +## High-level architecture + +- Entry point (`cmd/store/main.go`) + - Uses flags from `containerd/nydus-snapshotter` to parse config (`--root`, `--config-path`, logging), sets up `slog`. + - Builds a registry resolver with credentials from both Docker config and Podman `auth.json`. + - Initializes the `LayerManager`, mounts a FUSE filesystem under `/store`, and blocks until SIGINT. On exit, releases mounts. +- Resolver & Keychains (`pkg/services/...`) + - `resolver.RegistryHostsFromConfig` constructs `docker.RegistryHost` with retryable HTTP client and request timeouts. + - Credential sources: + - Docker: `pkg/services/keychain/dockerconfig` (supports identity token, user/pass; Docker Hub host normalized). + - Podman: `pkg/services/keychain/podmanauth` (searches `REGISTRY_AUTH_FILE`, XDG runtime, then `$HOME/.config/containers/auth.json`; supports identity token and base64 `auth`). +- Layer management (`pkg/manager`) + - Verifies signatures (configurable public key, optional validation). + - Spawns and tracks `nydusd` processes via snapshotter’s process manager; stores state in an embedded DB under ``. + - Resolves image manifests/configs, identifies Nydus meta layers, downloads bootstrap, mounts via Nydus FS, waits for readiness. + - Exposes mounted content by bind-mounting Nydus mountpoints to `/store///diff` (read-only), reference-counted per layer. + - Crash recovery: on startup, attempts to unmount any orphaned bind mounts found under `/store/*/*/diff`. + - OS-specific shims: Linux-specific mount helpers with `mount_shim_linux.go`, safe fallbacks in `mount_shim_other.go` for non-Linux builds. +- FUSE filesystem (`pkg/fs`) + - go-fuse v2 based; presents a structured view with directories and symlinks: `pool`, `diff`, `blob`, `info`, and `use` markers. + - Default permission modes are intentionally restrictive; can be overridden at mount time via `WithModes` (wired to CLI flags). + - Detects `fusermount`/`fusermount3`; if absent, attempts direct mount. Waits for server mount completion before returning. +- Integration with containers/storage + - `misc/storage.conf` declares `additionallayerstores = [ "/var/lib/nydus-store/store:ref" ]` to register the plugin’s store. + - Podman/CRI-O can then lazy-mount Nydus layers referenced by images. + +## Project rules for Warp agents + +- Use Go 1.25 toolchain. +- Maintain compatibility with Podman `auth.json` discovery and precedence; do not regress Docker config support. +- Prefer FUSE3 (`fusermount3`) when available; fallback paths must remain functional. +- Keep FS access modes configurable via CLI flags and plumbed through `pkg/fs`. +- Preserve and improve crash/unmount recovery semantics in `LayerManager` (e.g., `RecoverOrphanMounts`, `ReleaseAll`). + +## Notable files + +- `Makefile` — build and lint targets (`build`, `check`). +- `.golangci.yml` — enabled linters/formatters. +- `misc/nydus-config.json`, `misc/storage.conf` — sample runtime configs. +- `cmd/store/main.go` — CLI entry. +- `pkg/manager/*` — layer lifecycle, mounting, recovery. +- `pkg/fs/*` — FUSE filesystem and wiring. +- `pkg/services/{resolver,keychain}/` — registry access and auth. diff --git a/cmd/store/main.go b/cmd/store/main.go index d68cf2f..b13a94d 100644 --- a/cmd/store/main.go +++ b/cmd/store/main.go @@ -2,13 +2,14 @@ package main import ( "fmt" + "log/slog" "os" "os/signal" + "path/filepath" + "runtime" "syscall" - "github.com/containerd/containerd/log" "github.com/containerd/nydus-snapshotter/cmd/containerd-nydus-grpc/pkg/command" - "github.com/containerd/nydus-snapshotter/cmd/containerd-nydus-grpc/pkg/logging" "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/pkg/errdefs" "github.com/pkg/errors" @@ -17,6 +18,7 @@ import ( "github.com/containers/nydus-storage-plugin/pkg/fs" "github.com/containers/nydus-storage-plugin/pkg/manager" "github.com/containers/nydus-storage-plugin/pkg/services/keychain/dockerconfig" + podmanauth "github.com/containers/nydus-storage-plugin/pkg/services/keychain/podmanauth" "github.com/containers/nydus-storage-plugin/pkg/services/resolver" ) @@ -26,51 +28,150 @@ func waitForSIGINT() { <-c } +func parseOctalMode(s string) (uint32, error) { + var v uint32 + for i := 0; i < len(s); i++ { + c := s[i] + if c < '0' || c > '7' { + return 0, fmt.Errorf("invalid octal: %s", s) + } + v = (v << 3) | uint32(c-'0') + } + return v, nil +} + +func setupSlog(level string, toStdout bool, logDir string) error { + var lvl slog.Level + switch level { + case "debug": + lvl = slog.LevelDebug + case "warn", "warning": + lvl = slog.LevelWarn + case "error": + lvl = slog.LevelError + default: + lvl = slog.LevelInfo + } + var w *os.File + if toStdout || logDir == "" { + w = os.Stdout + } else { + path := filepath.Join(logDir, "nydus-store.log") + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + return err + } + w = f + } + h := slog.NewTextHandler(w, &slog.HandlerOptions{Level: lvl}) + slog.SetDefault(slog.New(h)) + return nil +} + func main() { flags := command.NewFlags() app := &cli.App{ Name: "crio nydus store", Usage: "crio nydus store plugin", Version: "0.0.0", - Flags: flags.F, - Action: func(c *cli.Context) error { - if err := logging.SetUp(flags.Args.LogLevel, flags.Args.LogToStdout, flags.Args.LogDir, flags.Args.RootDir); err != nil { + Flags: append(flags.F, + &cli.StringFlag{Name: "fs-file-mode", Usage: "octal file mode for files (e.g. 0400)"}, + &cli.StringFlag{Name: "fs-dir-mode", Usage: "octal dir mode (e.g. 0500)"}, + &cli.StringFlag{Name: "fs-link-mode", Usage: "octal symlink mode (e.g. 0400)"}, + &cli.BoolFlag{Name: "fs-allow-other", Value: true, Usage: "enable allow_other on FUSE mount"}, + &cli.BoolFlag{Name: "fs-direct-mount", Value: false, Usage: "force direct mount (bypass fusermount)"}, + &cli.BoolFlag{Name: "fs-mount-suid", Value: false, Usage: "add suid to fusermount mount options"}, + ), +Action: func(c *cli.Context) error { + if err := setupSlog(flags.Args.LogLevel, flags.Args.LogToStdout, flags.Args.LogDir); err != nil { return errors.Wrap(err, "failed to prepare logger") } + // Fail fast on unsupported platforms to avoid false-positive "mounted" states. + if runtime.GOOS != "linux" { + slog.ErrorContext(c.Context, "nydus-store requires Linux (FUSE) to mount; current OS unsupported", "GOOS", runtime.GOOS) + return errors.New("platform not supported: requires linux with FUSE") + } + var cfg config.Config if err := command.Validate(flags.Args, &cfg); err != nil { return errors.Wrap(err, "invalid argument") } mountPoint := fmt.Sprintf("%s/store", flags.Args.RootDir) - if _, err := os.Stat(mountPoint); err != nil { - if os.IsNotExist(err) { - if err := os.MkdirAll(mountPoint, 0755); err != nil { - return errors.Wrapf(err, "create root directory %s", mountPoint) - } - } else { - return errors.Wrapf(err, "stat root directory %s", mountPoint) - } + if err := os.MkdirAll(mountPoint, 0755); err != nil { + return errors.Wrapf(err, "create root directory %s", mountPoint) } // replace it with nydus-snapshotter resolver. - hosts := resolver.RegistryHostsFromConfig([]resolver.Credential{dockerconfig.NewDockerconfigKeychain(c.Context)}...) + hosts := resolver.RegistryHostsFromConfig( + []resolver.Credential{ + dockerconfig.NewDockerconfigKeychain(c.Context), + // Podman-compatible auth.json + podmanauth.NewPodmanAuthKeychain(c.Context), + }..., + ) layManager, err := manager.NewLayerManager(c.Context, flags.Args.RootDir, hosts, &cfg) if err != nil { panic(err) } - if err := fs.Mount(c.Context, mountPoint, flags.Args.RootDir, true, layManager); err != nil { - log.G(c.Context).WithError(err).Fatalf("failed to mount fs at %q", mountPoint) + // Parse optional FS modes + fileMode := fs.DefaultFileMode() + dirMode := fs.DefaultDirMode() + linkMode := fs.DefaultLinkMode() + if v := c.String("fs-file-mode"); v != "" { + if m, err := parseOctalMode(v); err == nil { + fileMode = m + } + } + if v := c.String("fs-dir-mode"); v != "" { + if m, err := parseOctalMode(v); err == nil { + dirMode = m + } + } + if v := c.String("fs-link-mode"); v != "" { + if m, err := parseOctalMode(v); err == nil { + linkMode = m + } + } + + // Recover orphan bind mounts from previous crashes + _ = layManager.RecoverOrphanMounts(c.Context) + + slog.InfoContext(c.Context, "Starting FUSE mount", + "mountPoint", mountPoint, + "rootDir", flags.Args.RootDir, + "fileMode", fmt.Sprintf("0%o", fileMode), + "dirMode", fmt.Sprintf("0%o", dirMode), + "linkMode", fmt.Sprintf("0%o", linkMode), + "allowOther", c.Bool("fs-allow-other")) + + if err := fs.Mount( + c.Context, + mountPoint, + flags.Args.RootDir, + true, + layManager, + fs.WithModes(fileMode, dirMode, linkMode), + fs.WithAllowOther(c.Bool("fs-allow-other")), + fs.WithDirectMount(c.Bool("fs-direct-mount")), + fs.WithMountSuid(c.Bool("fs-mount-suid")), + ); err != nil { + slog.ErrorContext(c.Context, "failed to mount fs", "mountPoint", mountPoint, "err", err) + return err } defer func() { + // Best-effort: release bind mounts first layManager.ReleaseAll(c.Context) - err := syscall.Unmount(mountPoint, 0) - if err != nil { - log.G(c.Context).Error(err) + // Try a normal unmount of the FUSE mountpoint + if err := syscall.Unmount(mountPoint, 0); err != nil { + slog.WarnContext(c.Context, "unmount busy; retry lazy unmount", "err", err) + if derr := lazyUnmount(mountPoint); derr != nil { + slog.ErrorContext(c.Context, "lazy unmount failed", "err", derr) + } } - log.G(c.Context).Info("Exiting") + slog.InfoContext(c.Context, "Exiting") }() waitForSIGINT() return nil @@ -78,9 +179,9 @@ func main() { } if err := app.Run(os.Args); err != nil { if errdefs.IsConnectionClosed(err) { - log.L.Info("snapshotter exited") + slog.Info("snapshotter exited") return } - log.L.WithError(err).Fatal("failed to start crio nydus store") + slog.Error("failed to start crio nydus store", "err", err) } } diff --git a/cmd/store/umount_linux.go b/cmd/store/umount_linux.go new file mode 100644 index 0000000..f548335 --- /dev/null +++ b/cmd/store/umount_linux.go @@ -0,0 +1,10 @@ +//go:build linux + +package main + +import "golang.org/x/sys/unix" + +// lazyUnmount performs a lazy unmount (MNT_DETACH) on Linux. +func lazyUnmount(target string) error { + return unix.Unmount(target, unix.MNT_DETACH) +} \ No newline at end of file diff --git a/cmd/store/umount_other.go b/cmd/store/umount_other.go new file mode 100644 index 0000000..75f20ae --- /dev/null +++ b/cmd/store/umount_other.go @@ -0,0 +1,6 @@ +//go:build !linux + +package main + +// lazyUnmount is a no-op fallback on non-Linux platforms. +func lazyUnmount(_ string) error { return nil } \ No newline at end of file diff --git a/go.mod b/go.mod index d82f6ac..07a30cf 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/containers/nydus-storage-plugin -go 1.18 +go 1.25 require ( github.com/containerd/containerd v1.6.19 diff --git a/pkg/cache/lrucache.go b/pkg/cache/lrucache.go index 9fc1eee..e7a909b 100644 --- a/pkg/cache/lrucache.go +++ b/pkg/cache/lrucache.go @@ -23,7 +23,7 @@ type LRUCache struct { // NewLRUCache creates new lru cache. func NewLRUCache(maxEntries int) *LRUCache { inner := lru.New(maxEntries) - inner.OnEvicted = func(key lru.Key, value interface{}) { + inner.OnEvicted = func(_ lru.Key, value interface{}) { // Decrease the ref count incremented in Add(). // When nobody refers to this value, this value will be finalized via refCounter. value.(*refCounter).finalize() diff --git a/pkg/fs/blob_file.go b/pkg/fs/blob_file.go index 7a7aa74..a042961 100644 --- a/pkg/fs/blob_file.go +++ b/pkg/fs/blob_file.go @@ -2,33 +2,111 @@ package fs import ( "context" + "io" + "log/slog" + "sync" "syscall" + "github.com/containerd/containerd/remotes/docker" fusefs "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// blob file is the file handle of blob contents. +// blobFile streams a remote blob and supports random access by reopening +// the stream and skipping to the requested offset when needed. type blobFile struct { + mu sync.Mutex + rc io.ReadCloser + size int64 + pos int64 + ref string + desc ocispec.Descriptor + hostsFn func(string) ([]docker.RegistryHost, error) +} + +func newBlobFile(_ context.Context, ref string, desc ocispec.Descriptor, hostsFn func(string) ([]docker.RegistryHost, error)) (*blobFile, error) { + return &blobFile{ + size: desc.Size, + pos: 0, + ref: ref, + desc: desc, + hostsFn: hostsFn, + }, nil } var _ = (fusefs.FileReader)((*blobFile)(nil)) +var _ = (fusefs.FileReleaser)((*blobFile)(nil)) +var _ = (fusefs.FileGetattrer)((*blobFile)(nil)) + +// openAt (re)opens the remote stream and skips to the specified offset. +func (f *blobFile) openAt(ctx context.Context, off int64) error { + if f.rc != nil { + _ = f.rc.Close() + f.rc = nil + } + resolver := docker.NewResolver(docker.ResolverOptions{ + Hosts: func(host string) ([]docker.RegistryHost, error) { return f.hostsFn(host) }, + }) + fetcher, err := resolver.Fetcher(ctx, f.ref) + if err != nil { + return err + } + r, err := fetcher.Fetch(ctx, f.desc) + if err != nil { + return err + } + // Skip to desired offset (naive sequential skip). Can be optimized with HTTP Range later. + if off > 0 { + if _, err := io.CopyN(io.Discard, r, off); err != nil { + _ = r.Close() + return err + } + } + f.rc = r + f.pos = off + return nil +} func (f *blobFile) Read(ctx context.Context, dest []byte, off int64) (fuse.ReadResult, syscall.Errno) { - //s, err := f.l.ReadAt(dest, off, - // remote.WithContext(ctx), // Make cancellable - // remote.WithCacheOpts(cache.Direct()), // Do not pollute mem cache - //) - //if err != nil && err != io.EOF { - // return nil, syscall.EIO - //} - //return fuse.ReadResultData(dest[:s]), 0 - return nil, syscall.EIO + f.mu.Lock() + defer f.mu.Unlock() + + // Guard against invalid handles (primarily for tests) and avoid panics. + if f.hostsFn == nil || f.ref == "" { + return nil, syscall.EIO + } + + if f.rc == nil || off != f.pos { + if err := f.openAt(ctx, off); err != nil { + slog.Warn("blob openAt failed", "off", off, "err", err) + return nil, syscall.EIO + } + } + if len(dest) == 0 { + return fuse.ReadResultData(nil), 0 + } + n, err := f.rc.Read(dest) + if n > 0 { + f.pos += int64(n) + } + if err != nil && err != io.EOF { + return nil, syscall.EIO + } + return fuse.ReadResultData(dest[:n]), 0 } -var _ = (fusefs.FileGetattrer)((*blobFile)(nil)) +func (f *blobFile) Release(_ context.Context) syscall.Errno { + f.mu.Lock() + defer f.mu.Unlock() + if f.rc != nil { + _ = f.rc.Close() + f.rc = nil + } + return 0 +} -func (f *blobFile) Getattr(ctx context.Context, out *fuse.AttrOut) syscall.Errno { - //layerToAttr(f.l, &out.Attr) +func (f *blobFile) Getattr(_ context.Context, out *fuse.AttrOut) syscall.Errno { + out.Size = uint64(f.size) return 0 } diff --git a/pkg/fs/blob_file_test.go b/pkg/fs/blob_file_test.go new file mode 100644 index 0000000..a432c07 --- /dev/null +++ b/pkg/fs/blob_file_test.go @@ -0,0 +1,23 @@ +package fs + +import ( + "context" + "testing" + + "github.com/hanwen/go-fuse/v2/fuse" +) + +func TestBlobFileReadEIO(t *testing.T) { + bf := &blobFile{} + if rr, eno := bf.Read(context.Background(), nil, 0); eno == 0 || rr != nil { + t.Fatalf("expected EIO and nil read result, got eno=%d rr=%v", eno, rr) + } +} + +func TestBlobFileGetattrNoError(t *testing.T) { + bf := &blobFile{} + var out fuse.AttrOut + if eno := bf.Getattr(context.Background(), &out); eno != 0 { + t.Fatalf("Getattr returned errno=%d", eno) + } +} diff --git a/pkg/fs/blob_node.go b/pkg/fs/blob_node.go index d3e8ffd..49a4efe 100644 --- a/pkg/fs/blob_node.go +++ b/pkg/fs/blob_node.go @@ -4,6 +4,8 @@ import ( "context" "syscall" + "github.com/containerd/containerd/reference" + "github.com/containerd/containerd/remotes/docker" fusefs "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -14,12 +16,24 @@ type blobNode struct { fusefs.Inode attr fuse.Attr l *ocispec.Descriptor + fs *fs + ref reference.Spec } var _ = (fusefs.InodeEmbedder)((*blobNode)(nil)) var _ = (fusefs.NodeOpener)((*blobNode)(nil)) -func (n *blobNode) Open(ctx context.Context, flags uint32) (fh fusefs.FileHandle, fuseFlags uint32, errno syscall.Errno) { - return &blobFile{}, 0, 0 +func (n *blobNode) Open(ctx context.Context, _ uint32) (fh fusefs.FileHandle, fuseFlags uint32, errno syscall.Errno) { + if n == nil || n.fs == nil || n.fs.layManager == nil || n.l == nil { + return nil, 0, syscall.EIO + } + hostsFn := func(_ string) ([]docker.RegistryHost, error) { + return n.fs.layManager.Hosts()(n.ref) + } + bf, err := newBlobFile(ctx, n.ref.String(), *n.l, hostsFn) + if err != nil { + return nil, 0, syscall.EIO + } + return bf, 0, 0 } diff --git a/pkg/fs/blob_node_test.go b/pkg/fs/blob_node_test.go new file mode 100644 index 0000000..60f8ccb --- /dev/null +++ b/pkg/fs/blob_node_test.go @@ -0,0 +1,14 @@ +package fs + +import ( + "context" + "testing" +) + +func TestBlobNodeOpenWithoutInitReturnsError(t *testing.T) { + n := &blobNode{} + _, _, eno := n.Open(context.Background(), 0) + if eno == 0 { + t.Fatalf("expected non-zero errno for uninitialized blobNode") + } +} diff --git a/pkg/fs/diff_node.go b/pkg/fs/diff_node.go index 9774b4c..7798bd6 100644 --- a/pkg/fs/diff_node.go +++ b/pkg/fs/diff_node.go @@ -15,7 +15,7 @@ type diffNode struct { fs *fs } -func (n *diffNode) Getattr(ctx context.Context, f fusefs.FileHandle, out *fuse.AttrOut) syscall.Errno { +func (n *diffNode) Getattr(_ context.Context, _ fusefs.FileHandle, out *fuse.AttrOut) syscall.Errno { copyAttr(&out.Attr, &n.attr) return 0 } diff --git a/pkg/fs/diff_node_test.go b/pkg/fs/diff_node_test.go new file mode 100644 index 0000000..26716a2 --- /dev/null +++ b/pkg/fs/diff_node_test.go @@ -0,0 +1,27 @@ +package fs + +import ( + "context" + "testing" + + "github.com/hanwen/go-fuse/v2/fuse" +) + +func TestDiffNodeGetattrCopiesAttr(t *testing.T) { + d := &diffNode{} + d.attr.Mode = 0o755 + var out fuse.AttrOut + if eno := d.Getattr(context.Background(), nil, &out); eno != 0 { + t.Fatalf("Getattr errno=%d", eno) + } + if out.Mode != d.attr.Mode { + t.Fatalf("mode mismatch: got %o want %o", out.Mode, d.attr.Mode) + } +} + +func TestDiffNodeRmdirAlwaysENOENT(t *testing.T) { + d := &diffNode{} + if eno := d.Rmdir(context.Background(), "anything"); eno == 0 { + t.Fatalf("expected ENOENT, got 0") + } +} diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 7675fbd..46bb2f4 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -9,28 +9,74 @@ import ( "syscall" "time" - "github.com/containerd/containerd/log" + "log/slog" + "github.com/containers/nydus-storage-plugin/pkg/manager" fusefs "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) const ( - defaultLinkMode = syscall.S_IFLNK | 0400 // -r-------- - defaultDirMode = syscall.S_IFDIR | 0500 // dr-x------ - defaultFileMode = 0400 // -r-------- - layerFileMode = 0400 // -r-------- - blockSize = 4096 + blockSize = 4096 poolLink = "pool" layerLink = "diff" blobLink = "blob" layerInfoLink = "info" layerUseFile = "use" +) + +// Configurable FS modes (defaults preserved) +var ( + // Permission bits only (no file type bits). File type is set via fuse StableAttr. + defaultLinkMode uint32 = 0400 // -r-------- (symlink perms are largely ignored by kernels) + defaultDirMode uint32 = 0500 // dr-x------ + defaultFileMode uint32 = 0400 // -r-------- + layerFileMode uint32 = 0400 // -r-------- - fusermountBin = "fusermount" + // Mount behavior toggles + defaultAllowOther bool = true + forceDirectMount bool = false + enableSuidOption bool = false ) +// Helpers to expose defaults for CLI parsing +func DefaultFileMode() uint32 { return defaultFileMode } +func DefaultDirMode() uint32 { return defaultDirMode } +func DefaultLinkMode() uint32 { return defaultLinkMode } + +// Mount options +type MountOption func() + +// WithModes overrides default FS modes at mount time. +func WithModes(fileMode, dirMode, linkMode uint32) MountOption { + return func() { + defaultFileMode = fileMode + defaultDirMode = dirMode + defaultLinkMode = linkMode + layerFileMode = fileMode + } +} + +// WithAllowOther toggles the allow_other mount option. +func WithAllowOther(b bool) MountOption { + return func() { + defaultAllowOther = b + } +} + +// WithDirectMount forces direct mount, bypassing fusermount helpers. +func WithDirectMount(b bool) MountOption { + return func() { + forceDirectMount = b + } +} + +// WithMountSuid toggles "suid" mount option when using fusermount. +func WithMountSuid(b bool) MountOption { + return func() { enableSuidOption = b } +} + type releasable interface { releasable() bool } @@ -78,7 +124,14 @@ func (r *inoReleasable) releasable() bool { return r.n.EmbeddedInode().Forgotten() } -func Mount(ctx context.Context, mountPoint string, rootDir string, debug bool, layManager *manager.LayerManager) error { +func Mount(_ context.Context, mountPoint string, _ string, debug bool, layManager *manager.LayerManager, opts ...MountOption) error { + // Apply mount options + for _, o := range opts { + if o != nil { + o() + } + } + seconds := time.Second rawFS := fusefs.NewNodeFS(&rootNode{ fs: &fs{ @@ -92,14 +145,23 @@ func Mount(ctx context.Context, mountPoint string, rootDir string, debug bool, l NullPermissions: true, }) mountOpts := &fuse.MountOptions{ - AllowOther: true, // allow users other than root&mounter to access fs + AllowOther: defaultAllowOther, // allow users other than root&mounter to access fs FsName: "nydusstore", Debug: debug, } - if _, err := exec.LookPath(fusermountBin); err == nil { - mountOpts.Options = []string{"suid"} // option for fusermount; allow setuid inside container + // Detect fusermount or fusermount3; fallback to direct mount if neither present + if hasFusermount() && !forceDirectMount { + if enableSuidOption { + mountOpts.Options = append(mountOpts.Options, "suid") // optional + } + slog.Info("using fusermount helper", "allowOther", mountOpts.AllowOther) } else { - log.L.WithError(err).Debugf("%s not installed; trying direct mount", fusermountBin) + if !hasFusermount() { + slog.Info("fusermount/fusermount3 not installed; trying direct mount") + } + if forceDirectMount { + slog.Info("forcing direct mount per option") + } mountOpts.DirectMount = true } server, err := fuse.NewServer(rawFS, mountPoint, mountOpts) @@ -110,13 +172,23 @@ func Mount(ctx context.Context, mountPoint string, rootDir string, debug bool, l return server.WaitMount() } +func hasFusermount() bool { + if _, err := exec.LookPath("fusermount"); err == nil { + return true + } + if _, err := exec.LookPath("fusermount3"); err == nil { + return true + } + return false +} + func (fs *fs) newInodeWithID(ctx context.Context, p func(uint32) fusefs.InodeEmbedder) (*fusefs.Inode, syscall.Errno) { var ino fusefs.InodeEmbedder if err := fs.nodeMap.add(func(id uint32) (releasable, error) { ino = p(id) return &inoReleasable{ino}, nil }); err != nil || ino == nil { - log.L.WithContext(ctx).WithError(err).Debug("cannot generate ID") + slog.DebugContext(ctx, "cannot generate ID", "err", err) return nil, syscall.EIO } return ino.EmbeddedInode(), 0 diff --git a/pkg/fs/fs_test.go b/pkg/fs/fs_test.go new file mode 100644 index 0000000..634f516 --- /dev/null +++ b/pkg/fs/fs_test.go @@ -0,0 +1,3 @@ +package fs + +// newInodeWithID depends on a live go-fuse server; skip unit testing here. diff --git a/pkg/fs/id_map.go b/pkg/fs/id_map.go index b3b646a..f6ebfd3 100644 --- a/pkg/fs/id_map.go +++ b/pkg/fs/id_map.go @@ -24,17 +24,17 @@ func (m *idMap) add(p func(uint32) (releasable, error)) error { m.cleanupG.Do("cleanup", func() (interface{}, error) { m.mu.Lock() defer m.mu.Unlock() - max := uint32(0) + maxID := uint32(0) for i := uint32(0); i <= m.max; i++ { if e, ok := m.m[i]; ok { if e.releasable() { delete(m.m, i) } else { - max = i + maxID = i } } } - m.max = max + m.max = maxID return nil, nil }) diff --git a/pkg/fs/id_map_test.go b/pkg/fs/id_map_test.go new file mode 100644 index 0000000..cc82138 --- /dev/null +++ b/pkg/fs/id_map_test.go @@ -0,0 +1,37 @@ +package fs + +import ( + "testing" +) + +type tr struct{ releasableFlag bool } + +func (r *tr) releasable() bool { return r.releasableFlag } + +func TestIDMapAddAssignsAndReuses(t *testing.T) { + m := &idMap{} + var saved []*tr + add := func(_ uint32) (releasable, error) { + r := &tr{} + saved = append(saved, r) + return r, nil + } + if err := m.add(add); err != nil { + t.Fatalf("add #1: %v", err) + } + if err := m.add(add); err != nil { + t.Fatalf("add #2: %v", err) + } + // IDs should start from 2 (skipping 0 and 1), so expect two entries + if len(m.m) != 2 { + t.Fatalf("expected 2 entries, got %d", len(m.m)) + } + // Mark first releasable and add again, expecting reuse of freed ID + saved[0].releasableFlag = true + if err := m.add(add); err != nil { + t.Fatalf("add #3: %v", err) + } + if len(m.m) != 2 { + t.Fatalf("expected still 2 entries after reuse, got %d", len(m.m)) + } +} diff --git a/pkg/fs/layer_node.go b/pkg/fs/layer_node.go index 970baba..431ddcf 100644 --- a/pkg/fs/layer_node.go +++ b/pkg/fs/layer_node.go @@ -6,9 +6,9 @@ import ( "bytes" "context" "encoding/json" + "log/slog" "syscall" - "github.com/containerd/containerd/log" fusefs "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" "github.com/opencontainers/go-digest" @@ -24,35 +24,34 @@ type layerNode struct { digest digest.Digest } -var _ = (fusefs.NodeGetattrer)((*diffNode)(nil)) -var _ = (fusefs.InodeEmbedder)((*diffNode)(nil)) var _ = (fusefs.InodeEmbedder)((*layerNode)(nil)) var _ = (fusefs.NodeCreater)((*layerNode)(nil)) var _ = (fusefs.NodeLookuper)((*layerNode)(nil)) +var _ = (fusefs.NodeReaddirer)((*layerNode)(nil)) // Create marks this layer as "using". // We don't use refnode.Mkdir because Mkdir event doesn't reach here if layernode already exists. -func (n *layerNode) Create(ctx context.Context, name string, flags uint32, mode uint32, out *fuse.EntryOut) (node *fusefs.Inode, fh fusefs.FileHandle, fuseFlags uint32, errno syscall.Errno) { +func (n *layerNode) Create(ctx context.Context, name string, _ uint32, _ uint32, _ *fuse.EntryOut) (node *fusefs.Inode, fh fusefs.FileHandle, fuseFlags uint32, errno syscall.Errno) { if name == layerUseFile { current := n.fs.layManager.Use(n.refNode.ref, n.digest) - log.G(ctx).WithField("refcounter", current).Infof("layer %v / %v is marked as USING", n.refNode.ref, n.digest) + slog.InfoContext(ctx, "layer marked USING", "ref", n.refNode.ref.String(), "digest", n.digest.String(), "refcounter", current) } return nil, nil, 0, syscall.ENOENT } // Lookup routes to the target file stored in the pool, based on the specified file name. func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fusefs.Inode, syscall.Errno) { - log.L.WithContext(ctx).Debugf("layer node lookup name = %s", name) + slog.InfoContext(ctx, "layer node lookup", "name", name) switch name { case layerInfoLink: info, err := n.fs.layManager.GetLayerInfo(ctx, n.refNode.ref, n.digest) if err != nil { - log.G(ctx).WithError(err).Warnf("failed to get layer info for %q: %q", name, n.digest) + slog.WarnContext(ctx, "failed to get layer info", "name", name, "digest", n.digest.String(), "err", err) return nil, syscall.EIO } buf := new(bytes.Buffer) if err := json.NewEncoder(buf).Encode(&info); err != nil { - log.G(ctx).WithError(err).Warnf("failed to encode layer info for %q: %q", name, n.digest) + slog.WarnContext(ctx, "failed to encode layer info", "name", name, "digest", n.digest.String(), "err", err) return nil, syscall.EIO } infoData := buf.Bytes() @@ -60,7 +59,7 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) cn := &fusefs.MemRegularFile{Data: infoData} copyAttr(&cn.Attr, &out.Attr) return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { - out.Attr.Ino = uint64(ino) + out.Ino = uint64(ino) cn.Attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) return n.NewInode(ctx, cn, sAttr) @@ -76,8 +75,8 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) copyAttr(&out.Attr, &ao.Attr) n.fs.knownNodeMu.Unlock() return n.NewInode(ctx, lh.n, fusefs.StableAttr{ - Mode: out.Attr.Mode, - Ino: out.Attr.Ino, + Mode: out.Mode, + Ino: out.Ino, }), 0 } n.fs.knownNodeMu.Unlock() @@ -85,27 +84,38 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) l, err := n.fs.layManager.ResolverMetaLayer(ctx, n.refNode.ref, n.refNode.rawRef, n.digest) if err != nil { - return nil, syscall.EIO + slog.WarnContext(ctx, "resolve meta layer failed", "err", err) + if name == layerLink { + return nil, syscall.ENOENT + } } - if name == blobLink { - sAttr := layerToAttr(l, &out.Attr) - cn := &blobNode{l: l} + sAttr := layerToAttr(&l.Descriptor, &out.Attr) + cn := &blobNode{l: &l.Descriptor, fs: n.fs, ref: n.refNode.ref} copyAttr(&cn.attr, &out.Attr) return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { - out.Attr.Ino = uint64(ino) + out.Ino = uint64(ino) cn.attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) return n.NewInode(ctx, cn, sAttr) }) } + // Only Nydus layers expose a diff directory. + // - bootstrap layer: diff is backed by a bind mount to the nydusd mountpoint + // - data blob layers: diff is an intentionally empty directory (to satisfy Podman additional layer store expectations) + if !l.IsMetaLayer || l.MountFailed { + slog.InfoContext(ctx, "not a nydus layer; no diff provided", "digest", n.digest.String()) + return nil, syscall.ENOENT + } + + sAttr := defaultDirAttr(&out.Attr) child := &diffNode{ fs: n.fs, } - sAttr := defaultDirAttr(&out.Attr) + copyAttr(&child.attr, &out.Attr) return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { - out.Attr.Ino = uint64(ino) + out.Ino = uint64(ino) child.attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) cn := n.NewInode(ctx, child, sAttr) @@ -123,10 +133,21 @@ func (n *layerNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) return cn }) case layerUseFile: - log.G(ctx).Debugf("\"use\" file is referred but return ENOENT for reference management") + slog.InfoContext(ctx, "use file referred; returning ENOENT for reference mgmt") return nil, syscall.ENOENT default: - log.G(ctx).Warnf("unknown filename %q", name) + slog.WarnContext(ctx, "unknown filename", "name", name) return nil, syscall.ENOENT } } + +// Readdir enumerates expected entries to help consumers like Podman discover files reliably. +func (n *layerNode) Readdir(_ context.Context) (fusefs.DirStream, syscall.Errno) { + entries := []fuse.DirEntry{ + {Name: layerInfoLink, Mode: fuse.S_IFREG}, + {Name: blobLink, Mode: fuse.S_IFREG}, + {Name: layerLink, Mode: fuse.S_IFDIR}, + {Name: layerUseFile, Mode: fuse.S_IFREG}, + } + return fusefs.NewListDirStream(entries), 0 +} diff --git a/pkg/fs/layer_node_more_test.go b/pkg/fs/layer_node_more_test.go new file mode 100644 index 0000000..3bdf322 --- /dev/null +++ b/pkg/fs/layer_node_more_test.go @@ -0,0 +1,17 @@ +package fs + +import ( + "context" + "testing" + + "github.com/hanwen/go-fuse/v2/fuse" +) + +func TestLayerNodeLookupUnknownNameENOENT(t *testing.T) { + n := &layerNode{fs: &fs{}} + var out fuse.EntryOut + _, eno := n.Lookup(context.Background(), "unknown", &out) + if eno == 0 { + t.Fatalf("expected ENOENT for unknown file name") + } +} diff --git a/pkg/fs/layer_node_test.go b/pkg/fs/layer_node_test.go new file mode 100644 index 0000000..b0dc078 --- /dev/null +++ b/pkg/fs/layer_node_test.go @@ -0,0 +1,41 @@ +package fs + +import ( + "context" + "testing" + + "github.com/hanwen/go-fuse/v2/fuse" +) + +func TestLayerNodeReaddir(t *testing.T) { + n := &layerNode{} + ds, errno := n.Readdir(context.Background()) + if errno != 0 { + t.Fatalf("Readdir returned errno=%d", errno) + } + + var names []string + var modes []uint32 + for ds.HasNext() { + de, _ := ds.Next() + names = append(names, de.Name) + modes = append(modes, de.Mode) + } + + expectNames := []string{layerInfoLink, blobLink, layerLink, layerUseFile} + if len(names) != len(expectNames) { + t.Fatalf("unexpected entries len: got %d, want %d", len(names), len(expectNames)) + } + for i, want := range expectNames { + if names[i] != want { + t.Errorf("entry %d name mismatch: got %q want %q", i, names[i], want) + } + } + // modes + expectModes := []uint32{fuse.S_IFREG, fuse.S_IFREG, fuse.S_IFDIR, fuse.S_IFREG} + for i, want := range expectModes { + if modes[i] != want { + t.Errorf("entry %d mode mismatch: got %v want %v", i, modes[i], want) + } + } +} diff --git a/pkg/fs/ref_node.go b/pkg/fs/ref_node.go index ddd01bd..602762b 100644 --- a/pkg/fs/ref_node.go +++ b/pkg/fs/ref_node.go @@ -4,9 +4,9 @@ package fs import ( "context" + "log/slog" "syscall" - "github.com/containerd/containerd/log" "github.com/containerd/containerd/reference" fusefs "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" @@ -28,21 +28,21 @@ var _ = (fusefs.NodeRmdirer)((*refNode)(nil)) func (n *refNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fusefs.Inode, syscall.Errno) { // lookup on memory nodes - log.L.WithContext(ctx).Debugf("ref node lookup name = %s", name) + slog.InfoContext(ctx, "ref node lookup", "name", name) if child := n.GetChild(name); child != nil { switch tn := child.Operations().(type) { case *layerNode: copyAttr(&out.Attr, &tn.attr) default: - log.G(ctx).Warn("rootnode.Lookup: uknown node type detected") + slog.WarnContext(ctx, "rootnode.Lookup: unknown node type detected") return nil, syscall.EIO } - out.Attr.Ino = child.StableAttr().Ino + out.Ino = child.StableAttr().Ino return child, 0 } targetDigest, err := digest.Parse(name) if err != nil { - log.G(ctx).WithError(err).Errorf("invalid digest for %q", name) + slog.ErrorContext(ctx, "invalid digest", "name", name, "err", err) return nil, syscall.EINVAL } sAttr := defaultDirAttr(&out.Attr) @@ -53,7 +53,7 @@ func (n *refNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) ( } copyAttr(&child.attr, &out.Attr) return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { - out.Attr.Ino = uint64(ino) + out.Ino = uint64(ino) child.attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) return n.NewInode(ctx, child, sAttr) @@ -63,12 +63,12 @@ func (n *refNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) ( func (n *refNode) Rmdir(ctx context.Context, name string) syscall.Errno { targetDigest, err := digest.Parse(name) if err != nil { - log.G(ctx).WithError(err).Warnf("invalid digest for %q during release", name) + slog.WarnContext(ctx, "invalid digest during release", "name", name, "err", err) return syscall.EINVAL } current, err := n.fs.layManager.Release(ctx, n.ref, targetDigest, n.rawRef) if err != nil { - log.G(ctx).WithError(err).Warnf("failed to release layer %v / %v", n.ref, targetDigest) + slog.WarnContext(ctx, "failed to release layer", "ref", n.ref.String(), "digest", targetDigest.String(), "err", err) return syscall.EIO } if current == 0 { @@ -76,7 +76,7 @@ func (n *refNode) Rmdir(ctx context.Context, name string) syscall.Errno { lh, ok := n.fs.knownNode[n.ref.String()][targetDigest.String()] if !ok { n.fs.knownNodeMu.Unlock() - log.G(ctx).WithError(err).Warnf("node of layer %v/%v is not registered", n.ref, targetDigest) + slog.WarnContext(ctx, "node of layer not registered", "ref", n.ref.String(), "digest", targetDigest.String(), "err", err) return syscall.EIO } lh.release() @@ -86,6 +86,6 @@ func (n *refNode) Rmdir(ctx context.Context, name string) syscall.Errno { } n.fs.knownNodeMu.Unlock() } - log.G(ctx).WithField("refcounter", current).Infof("layer %v/%v is marked as RELEASE", n.ref, targetDigest) + slog.InfoContext(ctx, "layer marked RELEASE", "ref", n.ref.String(), "digest", targetDigest.String(), "refcounter", current) return syscall.ENOENT } diff --git a/pkg/fs/ref_node_test.go b/pkg/fs/ref_node_test.go new file mode 100644 index 0000000..9c9d15b --- /dev/null +++ b/pkg/fs/ref_node_test.go @@ -0,0 +1,13 @@ +package fs + +import ( + "context" + "testing" +) + +func TestRefNodeRmdirInvalidDigest(t *testing.T) { + ref := refNode{fs: &fs{}} + if eno := ref.Rmdir(context.Background(), "not-a-digest"); eno == 0 { + t.Fatalf("expected error for invalid digest") + } +} diff --git a/pkg/fs/root_node.go b/pkg/fs/root_node.go index da38c1a..a288c99 100644 --- a/pkg/fs/root_node.go +++ b/pkg/fs/root_node.go @@ -5,14 +5,22 @@ package fs import ( "context" "encoding/base64" + "log/slog" + "strings" "syscall" - "github.com/containerd/containerd/log" "github.com/containerd/containerd/reference" fusefs "github.com/hanwen/go-fuse/v2/fs" "github.com/hanwen/go-fuse/v2/fuse" ) +func min(a, b int) int { + if a < b { + return a + } + return b +} + // rootnode is the mountpoint node of nydus-store. type rootNode struct { fusefs.Inode @@ -22,11 +30,13 @@ type rootNode struct { var _ = (fusefs.InodeEmbedder)((*rootNode)(nil)) var _ = (fusefs.NodeLookuper)((*rootNode)(nil)) +var _ = (fusefs.NodeReaddirer)((*rootNode)(nil)) +var _ = (fusefs.NodeUnlinker)((*rootNode)(nil)) // Lookup loads manifest and config of specified name (image reference) // and returns refnode of the specified name func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fusefs.Inode, syscall.Errno) { - log.L.WithContext(ctx).Debugf("root node lookup name = %s", name) + slog.InfoContext(ctx, "root node lookup", "name", name) if child := n.GetChild(name); child != nil { switch tn := child.Operations().(type) { case *fusefs.MemSymlink: @@ -34,10 +44,10 @@ func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) case *refNode: copyAttr(&out.Attr, &tn.attr) default: - log.L.WithContext(ctx).Warn("rootNode.Lookup: unknown node type detected") + slog.WarnContext(ctx, "rootNode.Lookup: unknown node type detected") return nil, syscall.EIO } - out.Attr.Ino = child.StableAttr().Ino + out.Ino = child.StableAttr().Ino return child, 0 } @@ -48,23 +58,53 @@ func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) cn := &fusefs.MemSymlink{Data: []byte(n.fs.layManager.RefRoot())} copyAttr(&cn.Attr, &out.Attr) return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { - out.Attr.Ino = uint64(ino) + out.Ino = uint64(ino) cn.Attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) return n.NewInode(ctx, cn, sAttr) }) } - refBytes, err := base64.StdEncoding.DecodeString(name) + // Handle system files (starting with .) gracefully + if strings.HasPrefix(name, ".") { + slog.DebugContext(ctx, "ignoring system file", "name", name) + return nil, syscall.ENOENT // File not found (better than EINVAL) + } + + // Try multiple base64 encodings + var refBytes []byte + var err error + + // Try standard base64 first + refBytes, err = base64.StdEncoding.DecodeString(name) if err != nil { - log.G(ctx).WithError(err).Errorf("failed to decode ref base64 %q", name) - return nil, syscall.EINVAL + // Try URL-safe base64 (without padding) + refBytes, err = base64.RawURLEncoding.DecodeString(name) + } + if err != nil { + // Try raw standard base64 (without padding) + refBytes, err = base64.RawStdEncoding.DecodeString(name) + } + + if err != nil { + slog.ErrorContext(ctx, "failed to decode base64 reference", + "name", name, + "name_len", len(name), + "err", err) + // Try to decode as much as possible for debugging + if len(name) >= 4 { + partial := name[:min(len(name), 32)] + if partialBytes, partialErr := base64.StdEncoding.DecodeString(partial + "==="); partialErr == nil { + slog.InfoContext(ctx, "partial decode result", "partial", partial, "decoded", string(partialBytes)) + } + } + return nil, syscall.ENOENT // File not found (better than EINVAL) } ref := string(refBytes) var refSpec reference.Spec refSpec, err = reference.Parse(ref) if err != nil { - log.G(ctx).WithError(err).Errorf("invalid reference %q for %q", ref, name) + slog.ErrorContext(ctx, "invalid reference", "ref", ref, "raw", name, "err", err) return nil, syscall.EINVAL } sAttr := defaultDirAttr(&out.Attr) @@ -75,9 +115,39 @@ func (n *rootNode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) } copyAttr(&child.attr, &out.Attr) return n.fs.newInodeWithID(ctx, func(ino uint32) fusefs.InodeEmbedder { - out.Attr.Ino = uint64(ino) + out.Ino = uint64(ino) child.attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) return n.NewInode(ctx, child, sAttr) }) } + +// Readdir enumerates entries in the root directory. +// Shows the "pool" symlink and any cached image references. +func (n *rootNode) Readdir(ctx context.Context) (fusefs.DirStream, syscall.Errno) { + // Start with the pool symlink + entries := []fuse.DirEntry{ + {Name: poolLink, Mode: fuse.S_IFLNK}, + {Name: "test-nydus-store-alive", Mode: fuse.S_IFREG}, + } + + // TODO: Add existing image references + // Currently, we only show the pool symlink since that's what's always available + + return fusefs.NewListDirStream(entries), 0 +} + +// Unlink prevents deletion of critical system entries like "pool" +func (n *rootNode) Unlink(ctx context.Context, name string) syscall.Errno { + slog.InfoContext(ctx, "root node unlink attempt", "name", name) + + // Prevent deletion of the pool symlink + if name == poolLink { + slog.WarnContext(ctx, "attempted to delete protected pool symlink", "name", name) + return syscall.EPERM // Operation not permitted + } + + // For other files, we don't support deletion + slog.InfoContext(ctx, "unlink not supported", "name", name) + return syscall.EPERM +} diff --git a/pkg/fs/root_node_test.go b/pkg/fs/root_node_test.go new file mode 100644 index 0000000..a7b9468 --- /dev/null +++ b/pkg/fs/root_node_test.go @@ -0,0 +1,3 @@ +package fs + +// Lookup paths depend on a running go-fuse server; covered by integration tests elsewhere. diff --git a/pkg/fs/utils.go b/pkg/fs/utils.go index afe921c..5ee499e 100644 --- a/pkg/fs/utils.go +++ b/pkg/fs/utils.go @@ -50,11 +50,11 @@ func layerToAttr(l *ocispec.Descriptor, out *fuse.Attr) fusefs.StableAttr { out.Blocks++ } out.Nlink = 1 - out.Mode = layerFileMode + out.Mode = fuse.S_IFREG | layerFileMode // include file type + perms out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ - Mode: out.Mode, + Mode: fuse.S_IFREG, // file type here } } @@ -68,10 +68,10 @@ func defaultFileAttr(size uint64, out *fuse.Attr) fusefs.StableAttr { out.Blocks++ } out.Nlink = 1 - out.Mode = defaultFileMode + out.Mode = fuse.S_IFREG | defaultFileMode // include file type + perms out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ - Mode: out.Mode, + Mode: fuse.S_IFREG, // file type here } } @@ -79,10 +79,10 @@ func defaultFileAttr(size uint64, out *fuse.Attr) fusefs.StableAttr { // https://github.com/containerd/stargz-snapshotter/blob/efc4166e93a22804b90e27c912eff7ecc0a12dfc/store/fs.go#L557 func defaultDirAttr(out *fuse.Attr) fusefs.StableAttr { out.Size = 0 - out.Mode = defaultDirMode + out.Mode = fuse.S_IFDIR | defaultDirMode // include dir type + perms out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ - Mode: out.Mode, + Mode: fuse.S_IFDIR, // directory type here } } @@ -90,9 +90,9 @@ func defaultDirAttr(out *fuse.Attr) fusefs.StableAttr { // https://github.com/containerd/stargz-snapshotter/blob/efc4166e93a22804b90e27c912eff7ecc0a12dfc/store/fs.go#L575 func defaultLinkAttr(out *fuse.Attr) fusefs.StableAttr { out.Size = 0 - out.Mode = defaultLinkMode + out.Mode = fuse.S_IFLNK | defaultLinkMode // include symlink type + perms out.Owner = fuse.Owner{Uid: 0, Gid: 0} return fusefs.StableAttr{ - Mode: out.Mode, + Mode: fuse.S_IFLNK, // symlink type here } } diff --git a/pkg/fs/utils_test.go b/pkg/fs/utils_test.go new file mode 100644 index 0000000..16668b4 --- /dev/null +++ b/pkg/fs/utils_test.go @@ -0,0 +1,48 @@ +package fs + +import ( + "testing" + + "github.com/hanwen/go-fuse/v2/fuse" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func TestUtilsCopyAttr(t *testing.T) { + var src, dst fuse.Attr + src.Ino = 123 + src.Size = 456 + src.Mode = 0o644 + src.Nlink = 2 + src.Owner = fuse.Owner{Uid: 1, Gid: 2} + copyAttr(&dst, &src) + if dst != src { + t.Fatalf("attr not copied: %+v != %+v", dst, src) + } +} + +func TestUtilsLayerToAttr(t *testing.T) { + var out fuse.Attr + s := layerToAttr(&ocispec.Descriptor{Size: 8192}, &out) + if out.Mode != (fuse.S_IFREG | layerFileMode) { + t.Fatalf("unexpected mode: %o", out.Mode) + } + if s.Mode != fuse.S_IFREG { + t.Fatalf("stable mode mismatch: got %o", s.Mode) + } +} + +func TestUtilsDefaultFileDirLinkAttr(t *testing.T) { + var out fuse.Attr + s1 := defaultFileAttr(100, &out) + if out.Mode != (fuse.S_IFREG | defaultFileMode) || s1.Mode != fuse.S_IFREG { + t.Fatalf("defaultFileAttr unexpected: out.Mode=%o stable.Mode=%o", out.Mode, s1.Mode) + } + s2 := defaultDirAttr(&out) + if out.Mode != (fuse.S_IFDIR | defaultDirMode) || s2.Mode != fuse.S_IFDIR { + t.Fatalf("defaultDirAttr unexpected: out.Mode=%o stable.Mode=%o", out.Mode, s2.Mode) + } + s3 := defaultLinkAttr(&out) + if out.Mode != (fuse.S_IFLNK | defaultLinkMode) || s3.Mode != fuse.S_IFLNK { + t.Fatalf("defaultLinkAttr unexpected: out.Mode=%o stable.Mode=%o", out.Mode, s3.Mode) + } +} diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 78b1822..e3228c1 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -3,13 +3,12 @@ package manager import ( "context" "fmt" + "log/slog" "os" - "os/exec" "path/filepath" "strings" "sync" - "github.com/containerd/containerd/log" "github.com/containerd/containerd/reference" "github.com/containerd/containerd/snapshots/storage" "github.com/containerd/nydus-snapshotter/config" @@ -25,11 +24,35 @@ import ( "github.com/containers/nydus-storage-plugin/pkg/source" ) +// fsDriver abstracts the subset of nydus filesystem methods we use, to enable testing. +// Implemented by *nydusFS.Filesystem. +type fsDriver interface { + UpperPath(id string) string + PrepareMetaLayer(ctx context.Context, snapshot storage.Snapshot, annotations map[string]string) error + Mount(ctx context.Context, snapshotID string, annotations map[string]string) error + WaitUntilReady(ctx context.Context, snapshotID string) error + MountPoint(snapshotID string) (string, error) +} + type nydusMessage struct { Err error } +type MountLayer struct { + Descriptor ocispec.Descriptor + IsMetaLayer bool + MountFailed bool +} + +var ErrMountMetaLayerFailed = errors.New("mount meta layer failed") + func NewLayerManager(ctx context.Context, rootDir string, hosts source.RegistryHosts, cfg *config.Config) (*LayerManager, error) { + slog.InfoContext(ctx, "NewLayerManager called", + "rootDir", rootDir, + "nydusdBinaryPath", cfg.NydusdBinaryPath, + "daemonMode", cfg.DaemonMode, + "cacheDir", cfg.CacheDir) + verifier, err := signature.NewVerifier(cfg.PublicKeyFile, cfg.ValidateSignature) if err != nil { return nil, err @@ -77,9 +100,6 @@ func NewLayerManager(ctx context.Context, rootDir string, hosts source.RegistryH if err != nil { return nil, err } - if err != nil { - return nil, fmt.Errorf("failed to setup resolver: %w", err) - } return &LayerManager{ refPool: refPool, hosts: hosts, @@ -94,10 +114,15 @@ type LayerManager struct { refPool *refPool hosts source.RegistryHosts - refCounter map[string]map[string]int - rootDir string + refCounter map[string]map[string]int + rootDir string + + // mountedSnapshots tracks snapshot IDs that already have an active nydusd mount. + mountedSnapshots sync.Map + // nydusMetaLayer tracks per-layer bind mounts (key: snapshotID+":"+layerDigest -> targetPath). nydusMetaLayer sync.Map - nydusFs *nydusFS.Filesystem + + nydusFs fsDriver mu sync.Mutex } @@ -110,19 +135,36 @@ func (r *LayerManager) GetLayerInfo(ctx context.Context, refspec reference.Spec, return genLayerInfo(dgst, manifest, imageConfig) } -func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference.Spec, snapshotID string, digest digest.Digest) (*ocispec.Descriptor, error) { +func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference.Spec, snapshotID string, digest digest.Digest) (*MountLayer, error) { + slog.InfoContext(ctx, "ResolverMetaLayer called", + "ref", refspec.String(), + "snapshotID", snapshotID, + "digest", digest.String()) + // get manifest from cache. manifest, _, err := r.refPool.loadRef(ctx, refspec) if err != nil { return nil, fmt.Errorf("failed to get manifest and config: %w", err) } + + slog.InfoContext(ctx, "manifest layers count", "count", len(manifest.Layers)) + var target ocispec.Descriptor var found bool - for _, l := range manifest.Layers { + for i, l := range manifest.Layers { + slog.InfoContext(ctx, "checking layer", + "index", i, + "digest", l.Digest.String(), + "mediaType", l.MediaType, + "annotations", l.Annotations) if l.Digest == digest { l := l found = true target = l + slog.InfoContext(ctx, "found target layer", + "digest", digest.String(), + "mediaType", l.MediaType, + "annotations", l.Annotations) break } } @@ -130,76 +172,142 @@ func (r *LayerManager) ResolverMetaLayer(ctx context.Context, refspec reference. return nil, fmt.Errorf("unknown digest %v for ref %q", target, refspec.String()) } - // Download nydus bootstrap layer and mount it. - if _, ok := target.Annotations[label.NydusMetaLayer]; ok { + layer := MountLayer{ + Descriptor: target, + IsMetaLayer: false, + } + + // Download nydus bootstrap/blob layer and mount it. + // NOTE: containers/image/nydus may set these annotations to non-"true" values; presence is what matters. + slog.InfoContext(ctx, "checking if layer is nydus layer", + "digest", target.Digest.String(), + "annotations", target.Annotations) + + isNydusLayer := false + isNydusBootstrap := false + isNydusBlob := false + if target.Annotations != nil { + if _, ok := target.Annotations[label.NydusMetaLayer]; ok { + isNydusLayer = true + isNydusBootstrap = true + } + // Legacy compatibility (same key as label.NydusMetaLayer in some older builds) + if _, ok := target.Annotations["containerd.io/snapshot/nydus-bootstrap"]; ok { + isNydusLayer = true + isNydusBootstrap = true + } + if _, ok := target.Annotations[label.NydusDataLayer]; ok { + isNydusLayer = true + isNydusBlob = true + } + } + + slog.InfoContext(ctx, "nydus layer check result", + "digest", target.Digest.String(), + "isNydusLayer", isNydusLayer, + "isNydusBootstrap", isNydusBootstrap, + "isNydusBlob", isNydusBlob, + "hasAnnotations", target.Annotations != nil) + + if target.Annotations != nil { + for k, v := range target.Annotations { + slog.InfoContext(ctx, "layer annotation", + "key", k, + "value", v) + } + } + + if isNydusLayer { target.Annotations[label.CRIImageRef] = refspec.String() target.Annotations[label.CRILayerDigest] = target.Digest.String() + layer.IsMetaLayer = true + + // For nydus data blob layers, we intentionally DO NOT provide a bind-mounted diff. + // Podman still requires the diff entry to exist, so the FUSE layer will serve an empty directory. + if isNydusBlob && !isNydusBootstrap { + slog.InfoContext(ctx, "nydus data blob layer detected; serving empty diff directory", + "digest", target.Digest.String(), + "snapshotID", snapshotID) + return &layer, nil + } - if _, ok = r.nydusMetaLayer.Load(snapshotID); ok { - log.G(ctx).Warnf("nydus duplicate mount meta layer ref is %s digest is %s", refspec.String(), target.Digest.String()) - return &target, nil + // Bootstrap layer: ensure nydusd mount exists and bind-mount its mountpoint to diff. + bindKey := snapshotID + ":" + target.Digest.String() + if _, exists := r.nydusMetaLayer.Load(bindKey); exists { + slog.DebugContext(ctx, "nydus duplicate bind mount", "ref", refspec.String(), "digest", target.Digest.String()) + return &layer, nil } workdir := r.nydusFs.UpperPath(snapshotID) if _, err = os.Stat(workdir); os.IsNotExist(err) { if err = os.MkdirAll(workdir, 0755); err != nil { - log.G(ctx).WithError(err).Error("mkdir nydus snapshot dir failed") - return nil, err + slog.ErrorContext(ctx, "mkdir nydus snapshot dir failed", "err", err) + return &layer, err } } - // Download nydus bootstrap layer to disk. - err = r.nydusFs.PrepareMetaLayer(ctx, storage.Snapshot{ID: snapshotID}, target.Annotations) - if err != nil && !strings.Contains(err.Error(), "file exists") { - log.G(ctx).WithError(err).Error("download snapshot files failed") - return nil, err - } + // Ensure the nydusd mount exists once per snapshotID. + if _, mounted := r.mountedSnapshots.Load(snapshotID); !mounted { + // Download nydus bootstrap to disk. + err = r.nydusFs.PrepareMetaLayer(ctx, storage.Snapshot{ID: snapshotID}, target.Annotations) + if err != nil && !strings.Contains(err.Error(), "file exists") { + slog.ErrorContext(ctx, "download snapshot files failed", "err", err) + return &layer, err + } - nydusMsgChannel := make(chan nydusMessage) + nydusMsgChannel := make(chan nydusMessage) - go func() { - log.G(ctx).Debugf("nydus mount meta layer ref is %s digest is %s", refspec.String(), target.Digest.String()) - err = r.nydusFs.Mount(ctx, snapshotID, target.Annotations) - if err != nil { - log.G(ctx).WithError(err).Error("nydus mount failed") - nydusMsgChannel <- nydusMessage{ - Err: err, + go func() { + slog.DebugContext(ctx, "nydus mount snapshot", "ref", refspec.String(), "digest", target.Digest.String(), "snapshotID", snapshotID) + err = r.nydusFs.Mount(ctx, snapshotID, target.Annotations) + if err != nil { + slog.ErrorContext(ctx, "nydus mount failed", "err", err) + nydusMsgChannel <- nydusMessage{Err: err} + return } - return - } + nydusMsgChannel <- nydusMessage{Err: nil} + }() - nydusMsgChannel <- nydusMessage{ - Err: nil, + event := <-nydusMsgChannel + if event.Err != nil { + return &layer, event.Err } - }() - event := <-nydusMsgChannel - if event.Err != nil { - return nil, event.Err - } + err = r.nydusFs.WaitUntilReady(ctx, snapshotID) + if err != nil { + return &layer, ErrMountMetaLayerFailed + } - err = r.nydusFs.WaitUntilReady(ctx, snapshotID) - if err != nil { - return nil, err + r.mountedSnapshots.Store(snapshotID, true) } - // Link nydusd mount dir to /// - targetPath := fmt.Sprintf("%s/store/%s/%s/diff", r.rootDir, snapshotID, target.Digest.String()) + // Link nydusd mount dir to ///diff + targetPath := filepath.Join(r.rootDir, "store", snapshotID, target.Digest.String(), "diff") var mountPoint string if mountPoint, err = r.nydusFs.MountPoint(snapshotID); err == nil { - cmd := exec.Command("mount", "-o", "bind,ro", mountPoint, targetPath) - if err = cmd.Start(); err == nil { - r.nydusMetaLayer.Store(snapshotID, targetPath) - return &target, nil + if err := os.MkdirAll(targetPath, 0755); err != nil { + slog.ErrorContext(ctx, "ensure targetPath failed", "err", err) + return &layer, err + } + // Perform a bind mount and then remount read-only for robustness across kernels + if err = unixMount(mountPoint, targetPath, "", msBind|msRec, ""); err != nil { + slog.ErrorContext(ctx, "bind mount failed", "err", err) + return &layer, err + } + if err = unixMount("", targetPath, "", msBind|msRemount|msRdonly, ""); err != nil { + slog.ErrorContext(ctx, "remount ro failed", "err", err) + // try to unmount in case remount failed partially + _ = unixUnmount(targetPath, 0) + return &layer, err } - log.G(ctx).WithError(err).Error("mount bind file has error") - return nil, err + r.nydusMetaLayer.Store(bindKey, targetPath) + return &layer, nil } - log.G(ctx).WithError(err).Error("get mount point failed") - return nil, err + slog.ErrorContext(ctx, "get mount point failed", "err", err) + return &layer, err } // TODO support normal image format. - return &target, nil + return &layer, nil } func (r *LayerManager) Release(ctx context.Context, refspec reference.Spec, dgst digest.Digest, snapshotID string) (int, error) { @@ -218,35 +326,69 @@ func (r *LayerManager) Release(ctx context.Context, refspec reference.Spec, dgst r.refCounter[refspec.String()][dgst.String()]-- i := r.refCounter[refspec.String()][dgst.String()] if i <= 0 { - if v, ok := r.nydusMetaLayer.Load(snapshotID); ok { - cmd := exec.Command("umount", v.(string)) - if err := cmd.Run(); err != nil { - log.G(ctx).Errorf("umount bind nydus %v/%v failed: %+v", refspec, dgst, err) + bindKey := snapshotID + ":" + dgst.String() + if v, ok := r.nydusMetaLayer.Load(bindKey); ok { + if err := unixUnmount(v.(string), 0); err != nil { + slog.ErrorContext(ctx, "umount bind nydus failed", "ref", refspec.String(), "digest", dgst.String(), "err", err) return 0, err } - r.nydusMetaLayer.Delete(snapshotID) + r.nydusMetaLayer.Delete(bindKey) } - // No reference to this layer. release it. - delete(r.refCounter, dgst.String()) + delete(r.refCounter[refspec.String()], dgst.String()) if len(r.refCounter[refspec.String()]) == 0 { delete(r.refCounter, refspec.String()) } - log.G(ctx).WithField("refcounter", i).Infof("layer %v/%v is released due to no reference", refspec, dgst) + slog.InfoContext(ctx, "layer released due to no reference", "ref", refspec.String(), "digest", dgst.String(), "refcounter", i) } return i, nil } func (r *LayerManager) ReleaseAll(ctx context.Context) { r.nydusMetaLayer.Range(func(key, value interface{}) bool { - cmd := exec.Command("umount", value.(string)) - if err := cmd.Run(); err != nil { - log.G(ctx).WithError(err).Warnf("umount bind nydus %v/%v failed", key, value) + if err := unixUnmount(value.(string), 0); err != nil { + slog.WarnContext(ctx, "umount bind nydus failed", "key", key, "value", value, "err", err) } return true }) } +// RecoverOrphanMounts attempts to unmount any bind-mounted diff directories +// left over from previous crashes. It walks /store/*/*/diff and tries to unmount. +func (r *LayerManager) RecoverOrphanMounts(ctx context.Context) error { + storeRoot := filepath.Join(r.rootDir, "store") + slog.InfoContext(ctx, "RecoverOrphanMounts checking store", "storeRoot", storeRoot) + + ents, err := os.ReadDir(storeRoot) + if err != nil { + if os.IsNotExist(err) { + slog.InfoContext(ctx, "store directory does not exist, skipping recovery", "storeRoot", storeRoot) + return nil + } + return err + } + + slog.InfoContext(ctx, "found entries in store", "count", len(ents), "storeRoot", storeRoot) + for _, e := range ents { + if !e.IsDir() { + continue + } + refDir := filepath.Join(storeRoot, e.Name()) + layers, _ := os.ReadDir(refDir) + for _, l := range layers { + if !l.IsDir() { + continue + } + diff := filepath.Join(refDir, l.Name(), "diff") + // Best-effort unmount + if err := unixUnmount(diff, 0); err == nil { + slog.DebugContext(ctx, "recovered orphan mount", "path", diff) + } + } + } + return nil +} + func (r *LayerManager) Use(refspec reference.Spec, dgst digest.Digest) int { r.refPool.use(refspec) @@ -271,6 +413,9 @@ func (r *LayerManager) RefRoot() string { return r.refPool.root() } +// Hosts returns the registry hosts provider. +func (r *LayerManager) Hosts() source.RegistryHosts { return r.hosts } + func colon2dash(s string) string { return strings.ReplaceAll(s, ":", "-") } diff --git a/pkg/manager/manager_test.go b/pkg/manager/manager_test.go new file mode 100644 index 0000000..d3483ba --- /dev/null +++ b/pkg/manager/manager_test.go @@ -0,0 +1,190 @@ +package manager + +import ( + "context" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/containerd/containerd/reference" + "github.com/containerd/containerd/snapshots/storage" + "github.com/containerd/nydus-snapshotter/pkg/label" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +type fakeFS struct { + root string + mp string +} + +func (f *fakeFS) UpperPath(id string) string { return filepath.Join(f.root, "snapshots", id) } +func (f *fakeFS) PrepareMetaLayer(_ context.Context, _ storage.Snapshot, _ map[string]string) error { + return nil +} +func (f *fakeFS) Mount(_ context.Context, _ string, _ map[string]string) error { + return nil +} +func (f *fakeFS) WaitUntilReady(_ context.Context, _ string) error { return nil } +func (f *fakeFS) MountPoint(_ string) (string, error) { return f.mp, nil } + +func TestResolverMetaLayerBindMountAndRemountRO(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + + // prepare manifest+config in refPool + p, err := newRefPool(ctx, dir, nil) + if err != nil { + t.Fatalf("newRefPool: %v", err) + } + refspec, _ := reference.Parse("docker.io/library/busybox:latest") + dgst := digest.FromString("layer-1") + manifest := ocispec.Manifest{Layers: []ocispec.Descriptor{{ + MediaType: ocispec.MediaTypeImageLayerGzip, + Digest: dgst, + Size: 10, + Annotations: map[string]string{label.NydusMetaLayer: "true"}, + }}, Config: ocispec.Descriptor{MediaType: ocispec.MediaTypeImageConfig, Digest: digest.FromString("cfg"), Size: 1}} + config := ocispec.Image{RootFS: ocispec.RootFS{Type: "layers", DiffIDs: []digest.Digest{digest.FromString("diffid")}}} + if err := p.writeManifestAndConfig(refspec, manifest, config); err != nil { + t.Fatalf("writeManifestAndConfig: %v", err) + } + + mountPoint := filepath.Join(dir, "nydus-mp") + if err := os.MkdirAll(mountPoint, 0o755); err != nil { + t.Fatalf("mkdir mountpoint: %v", err) + } + + lm := &LayerManager{ + refPool: p, + refCounter: make(map[string]map[string]int), + nydusFs: &fakeFS{root: dir, mp: mountPoint}, + rootDir: dir, + } + + // stub unix mount + origMount := unixMount + defer func() { unixMount = origMount }() + type mcall struct { + src, tgt, fstype, data string + flags uintptr + } + var calls []mcall + unixMount = func(source, target, fstype string, flags uintptr, data string) error { + calls = append(calls, mcall{source, target, fstype, data, flags}) + return nil + } + + snapshotID := "snap-1" + layer, err := lm.ResolverMetaLayer(ctx, refspec, snapshotID, dgst) + if err != nil { + t.Fatalf("ResolverMetaLayer error: %v", err) + } + if !layer.IsMetaLayer { + t.Fatalf("expected IsMetaLayer=true") + } + + targetPath := filepath.Join(dir, "store", snapshotID, dgst.String(), "diff") + if _, err := os.Stat(targetPath); err != nil { + t.Fatalf("targetPath not created: %v", err) + } + + if len(calls) != 2 { + t.Fatalf("expected 2 mount calls, got %d", len(calls)) + } + if calls[0].src != mountPoint || calls[0].tgt != targetPath { + t.Errorf("bind mount args mismatch: got (%q,%q), want (%q,%q)", calls[0].src, calls[0].tgt, mountPoint, targetPath) + } + if calls[1].src != "" || calls[1].tgt != targetPath { + t.Errorf("remount args mismatch: got (%q,%q)", calls[1].src, calls[1].tgt) + } + // flags + if calls[0].flags&(0x1000|0x4000) == 0 { // MS_BIND|MS_REC + t.Errorf("bind mount flags missing MS_BIND|MS_REC: %#x", calls[0].flags) + } + if calls[1].flags&(0x1000|0x20|0x1) == 0 { // MS_BIND|MS_REMOUNT|MS_RDONLY + t.Errorf("remount flags missing: %#x", calls[1].flags) + } +} + +func TestResolverMetaLayerCreatesTargetDir(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + p, err := newRefPool(ctx, dir, nil) + if err != nil { + t.Fatalf("newRefPool: %v", err) + } + refspec, _ := reference.Parse("docker.io/library/busybox:latest") + dgst := digest.FromString("layer-2") + manifest := ocispec.Manifest{Layers: []ocispec.Descriptor{{ + MediaType: ocispec.MediaTypeImageLayerGzip, + Digest: dgst, + Size: 10, + Annotations: map[string]string{label.NydusMetaLayer: "true"}, + }}, Config: ocispec.Descriptor{MediaType: ocispec.MediaTypeImageConfig, Digest: digest.FromString("cfg2"), Size: 1}} + config := ocispec.Image{RootFS: ocispec.RootFS{Type: "layers", DiffIDs: []digest.Digest{digest.FromString("diffid2")}}} + if err := p.writeManifestAndConfig(refspec, manifest, config); err != nil { + t.Fatalf("writeManifestAndConfig: %v", err) + } + mountPoint := filepath.Join(dir, "nydus-mp2") + if err := os.MkdirAll(mountPoint, 0o755); err != nil { + t.Fatalf("mkdir mountpoint: %v", err) + } + lm := &LayerManager{refPool: p, refCounter: make(map[string]map[string]int), nydusFs: &fakeFS{root: dir, mp: mountPoint}, rootDir: dir} + origMount := unixMount + defer func() { unixMount = origMount }() + unixMount = func(_, _, _ string, _ uintptr, _ string) error { return nil } + snapshotID := "snap-2" + _, err = lm.ResolverMetaLayer(ctx, refspec, snapshotID, dgst) + if err != nil { + t.Fatalf("ResolverMetaLayer error: %v", err) + } + targetPath := filepath.Join(dir, "store", snapshotID, dgst.String(), "diff") + if st, err := os.Stat(targetPath); err != nil || !st.IsDir() { + t.Fatalf("targetPath not created as dir: %v, st=%v", err, st) + } +} + +func TestReleaseDecrementsAndUnmountsAndCleansMaps(t *testing.T) { + ctx := context.Background() + lm := &LayerManager{ + refPool: &refPool{refcounter: map[string]*releaser{}}, + refCounter: map[string]map[string]int{}, + } + refspec, _ := reference.Parse("docker.io/library/busybox:latest") + dgst := digest.FromString("layer-1") + snapshotID := "snap-1" + // prepare counters + lm.refCounter[refspec.String()] = map[string]int{dgst.String(): 1} + lm.nydusMetaLayer.Store(snapshotID+":"+dgst.String(), "/fake/target") + lm.refPool.refcounter[refspec.String()] = &releaser{count: 1, release: func() {}} + + var unmounted []string + origUnmount := unixUnmount + defer func() { unixUnmount = origUnmount }() + unixUnmount = func(target string, _ int) error { + unmounted = append(unmounted, target) + return nil + } + + i, err := lm.Release(ctx, refspec, dgst, snapshotID) + if err != nil { + t.Fatalf("Release error: %v", err) + } + if i != 0 { + t.Fatalf("expected return 0, got %d", i) + } + if !reflect.DeepEqual(unmounted, []string{"/fake/target"}) { + t.Errorf("unexpected unmounts: %#v", unmounted) + } + if _, ok := lm.refCounter[refspec.String()][dgst.String()]; ok { + t.Errorf("layer entry not removed from refCounter") + } + if _, ok := lm.refCounter[refspec.String()]; ok { + t.Errorf("ref entry not removed from refCounter") + } + if _, ok := lm.nydusMetaLayer.Load(snapshotID + ":" + dgst.String()); ok { + t.Errorf("nydusMetaLayer entry not deleted") + } +} diff --git a/pkg/manager/mount_shim_linux.go b/pkg/manager/mount_shim_linux.go new file mode 100644 index 0000000..6870a69 --- /dev/null +++ b/pkg/manager/mount_shim_linux.go @@ -0,0 +1,24 @@ +//go:build linux + +package manager + +import "golang.org/x/sys/unix" + +const ( + msBind = unix.MS_BIND + msRec = unix.MS_REC + msRemount = unix.MS_REMOUNT + msRdonly = unix.MS_RDONLY +) + +type mountFunc func(source, target, fstype string, flags uintptr, data string) error + +type unmountFunc func(target string, flags int) error + +var unixMount mountFunc = func(source, target, fstype string, flags uintptr, data string) error { + return unix.Mount(source, target, fstype, flags, data) +} + +var unixUnmount unmountFunc = func(target string, flags int) error { + return unix.Unmount(target, flags) +} diff --git a/pkg/manager/mount_shim_other.go b/pkg/manager/mount_shim_other.go new file mode 100644 index 0000000..0d6cff3 --- /dev/null +++ b/pkg/manager/mount_shim_other.go @@ -0,0 +1,26 @@ +//go:build !linux + +package manager + +import "errors" + +const ( + // Linux mount flags used for tests; values chosen to match linux for assertions. + msBind = 0x1000 + msRec = 0x4000 + msRemount = 0x20 + msRdonly = 0x1 +) + +type mountFunc func(source, target, fstype string, flags uintptr, data string) error + +type unmountFunc func(target string, flags int) error + +// On non-linux, return explicit errors by default so production runs fail fast. +// Unit tests replace these vars to stub platform-specific behavior. +var unixMount mountFunc = func(_, _, _ string, _ uintptr, _ string) error { + return errors.New("mount is not supported on non-linux platforms") +} +var unixUnmount unmountFunc = func(_ string, _ int) error { + return errors.New("unmount is not supported on non-linux platforms") +} diff --git a/pkg/manager/ref_pool.go b/pkg/manager/ref_pool.go index 16ca6ce..314408a 100644 --- a/pkg/manager/ref_pool.go +++ b/pkg/manager/ref_pool.go @@ -7,13 +7,13 @@ import ( "encoding/json" "fmt" "io" + "log/slog" "os" "path/filepath" "sync" "time" "github.com/containerd/containerd/images" - "github.com/containerd/containerd/log" "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/reference" "github.com/containerd/containerd/remotes" @@ -23,7 +23,7 @@ import ( "github.com/containers/nydus-storage-plugin/pkg/cache" "github.com/containers/nydus-storage-plugin/pkg/source" - "github.com/containers/nydus-storage-plugin/pkg/utils" + manifestutil "github.com/containers/nydus-storage-plugin/pkg/utils" ) const ( @@ -45,10 +45,10 @@ func newRefPool(ctx context.Context, root string, hosts source.RegistryHosts) (* p.cache.OnEvicted = func(key string, value interface{}) { refspec := value.(reference.Spec) if err := os.RemoveAll(p.metadataDir(refspec)); err != nil { - log.G(ctx).WithField("key", key).WithError(err).Warnf("failed to clean up ref") + slog.WarnContext(ctx, "failed to clean up ref", "key", key, "err", err) return } - log.G(ctx).WithField("key", key).Debugf("cleaned up ref") + slog.DebugContext(ctx, "cleaned up ref", "key", key) } return p, nil } @@ -70,10 +70,10 @@ type releaser struct { func (p *refPool) loadRef(ctx context.Context, refspec reference.Spec) (manifest ocispec.Manifest, config ocispec.Image, err error) { manifest, config, err = p.readManifestAndConfig(refspec) if err == nil { - log.G(ctx).Debugf("reusing manifest and config of %q", refspec.String()) + slog.DebugContext(ctx, "reusing manifest and config", "ref", refspec.String()) return } - log.G(ctx).WithError(err).Debugf("fetching manifest and config of %q", refspec.String()) + slog.DebugContext(ctx, "fetching manifest and config", "ref", refspec.String(), "err", err) manifest, config, err = p.fetchManifestAndConfig(ctx, refspec) if err != nil { return ocispec.Manifest{}, ocispec.Image{}, err @@ -152,7 +152,7 @@ func (p *refPool) readManifestAndConfig(refspec reference.Spec) (manifest ocispe func (p *refPool) writeManifestAndConfig(refspec reference.Spec, manifest ocispec.Manifest, config ocispec.Image) error { mPath, cPath := p.manifestFile(refspec), p.configFile(refspec) - log.G(context.TODO()).Infof("mpath = %s, cpath = %s", mPath, cPath) + slog.Info("write manifest and config paths", "manifest", mPath, "config", cPath) if err := os.MkdirAll(filepath.Dir(mPath), 0700); err != nil { return err } @@ -245,7 +245,7 @@ func fetchManifestPlatform(ctx context.Context, fetcher remotes.Fetcher, desc oc if err != nil { return ocispec.Manifest{}, err } - if err := utils.ValidateMediaType(p, desc.MediaType); err != nil { + if err := manifestutil.ValidateMediaType(p, desc.MediaType); err != nil { return ocispec.Manifest{}, err } if err := json.Unmarshal(p, &manifest); err != nil { @@ -258,7 +258,7 @@ func fetchManifestPlatform(ctx context.Context, fetcher remotes.Fetcher, desc oc if err != nil { return ocispec.Manifest{}, err } - if err := utils.ValidateMediaType(p, desc.MediaType); err != nil { + if err := manifestutil.ValidateMediaType(p, desc.MediaType); err != nil { return ocispec.Manifest{}, err } if err = json.Unmarshal(p, &index); err != nil { diff --git a/pkg/services/keychain/dockerconfig/dockerconfig.go b/pkg/services/keychain/dockerconfig/dockerconfig.go index d657d5f..0e690ba 100644 --- a/pkg/services/keychain/dockerconfig/dockerconfig.go +++ b/pkg/services/keychain/dockerconfig/dockerconfig.go @@ -2,8 +2,8 @@ package dockerconfig import ( "context" + "log/slog" - "github.com/containerd/containerd/log" "github.com/containerd/containerd/reference" "github.com/docker/cli/cli/config" @@ -13,10 +13,10 @@ import ( // Ported from stargz-snapshotter, copyright The stargz-snapshotter Authors. // https://github.com/containerd/stargz-snapshotter/blob/923399007a8cde1ec871072ba6678b428b40b852/service/keychain/dockerconfig/dockerconfig.go func NewDockerconfigKeychain(ctx context.Context) resolver.Credential { - return func(host string, refspec reference.Spec) (string, string, error) { + return func(host string, _ reference.Spec) (string, string, error) { cf, err := config.Load("") if err != nil { - log.G(ctx).WithError(err).Warnf("failed to load docker config file") + slog.WarnContext(ctx, "failed to load docker config file", "err", err) return "", "", nil } diff --git a/pkg/services/keychain/podmanauth/podmanauth.go b/pkg/services/keychain/podmanauth/podmanauth.go new file mode 100644 index 0000000..01e15ef --- /dev/null +++ b/pkg/services/keychain/podmanauth/podmanauth.go @@ -0,0 +1,110 @@ +package podmanauth + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/reference" + + "github.com/containers/nydus-storage-plugin/pkg/services/resolver" +) + +// NewPodmanAuthKeychain returns a resolver.Credential that sources credentials +// from Podman-compatible auth.json files. +// Precedence: +// 1) REGISTRY_AUTH_FILE (path to an auth.json) +// 2) $XDG_RUNTIME_DIR/containers/auth.json +// 3) $HOME/.config/containers/auth.json +func NewPodmanAuthKeychain(ctx context.Context) resolver.Credential { + return func(host string, _ reference.Spec) (string, string, error) { + path, err := findAuthFile() + if err != nil || path == "" { + return "", "", nil + } + creds, err := readAuthFile(path) + if err != nil { + log.G(ctx).WithError(err).Warnf("failed to read podman auth.json from %s", path) + return "", "", nil + } + + // Docker Hub special-case compatibility + if host == "docker.io" || host == "registry-1.docker.io" { + host = "https://index.docker.io/v1/" + } + + if e, ok := creds.Auths[host]; ok { + if e.IdentityToken != "" { + return "", e.IdentityToken, nil + } + // Prefer explicit username/password if present + if e.Username != "" || e.Password != "" { + return e.Username, e.Password, nil + } + if e.Auth != "" { + b, err := base64.StdEncoding.DecodeString(e.Auth) + if err == nil { + p := string(b) + if idx := strings.IndexByte(p, ':'); idx >= 0 { + return p[:idx], p[idx+1:], nil + } + } + } + } + return "", "", nil + } +} + +// Minimal struct for containers-auth.json +type authFile struct { + Auths map[string]authEntry `json:"auths"` +} + +type authEntry struct { + Auth string `json:"auth"` + Username string `json:"username"` + Password string `json:"password"` + IdentityToken string `json:"identitytoken"` +} + +func findAuthFile() (string, error) { + if p := os.Getenv("REGISTRY_AUTH_FILE"); p != "" { + return p, nil + } + if xdg := os.Getenv("XDG_RUNTIME_DIR"); xdg != "" { + p := filepath.Join(xdg, "containers", "auth.json") + if _, err := os.Stat(p); err == nil { + return p, nil + } + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + p := filepath.Join(home, ".config", "containers", "auth.json") + if _, err := os.Stat(p); err == nil { + return p, nil + } + return "", nil +} + +func readAuthFile(path string) (authFile, error) { + f, err := os.Open(path) + if err != nil { + return authFile{}, err + } + defer f.Close() + var af authFile + if err := json.NewDecoder(f).Decode(&af); err != nil { + return authFile{}, err + } + if af.Auths == nil { + return authFile{}, errors.New("no auths") + } + return af, nil +} diff --git a/pkg/services/resolver/resolver.go b/pkg/services/resolver/resolver.go index 88ab017..952287c 100644 --- a/pkg/services/resolver/resolver.go +++ b/pkg/services/resolver/resolver.go @@ -83,7 +83,7 @@ func multiCredsFuncs(ref reference.Spec, credsFuncs ...Credential) func(string) for _, f := range credsFuncs { if username, secret, err := f(host, ref); err != nil { return "", "", err - } else if !(username == "" && secret == "") { + } else if username != "" || secret != "" { return username, secret, nil } } diff --git a/pkg/utils/manifest.go b/pkg/utils/manifest.go index e2979e4..bcaf286 100644 --- a/pkg/utils/manifest.go +++ b/pkg/utils/manifest.go @@ -1,6 +1,6 @@ // Ported from stargz-snapshotter, copyright The stargz-snapshotter Authors. // https://github.com/containerd/stargz-snapshotter/blob/6fb41553e735eb6369bb3718d4b841bfacb423aa/util/containerdutil/manifest.go#L119-L158 -package utils +package manifestutil import ( "encoding/json"