Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion storage/docs/containers-storage-dedup.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ containers-storage-dedup - Deduplicate similar files in the images
**containers-storage** **dedup**

## DESCRIPTION
Find similar files in the images and deduplicate them. It requires reflink support from the file system.
Find similar files in the images and deduplicate them. It requires reflink support from the file system.

To check if your filesystem supports reflinks, run **containers-storage status** and look for the **Supports reflinks** field. Deduplication will only work on filesystems that report reflink support (Btrfs, XFS with reflink=1, OCFS2).

## OPTIONS
**--hash-method** *method*
Expand All @@ -17,3 +19,6 @@ Specify the function to use to calculate the hash for a file. It can be one of:
## EXAMPLE

containers-storage dedup

## SEE ALSO
containers-storage-status(1)
4 changes: 3 additions & 1 deletion storage/docs/containers-storage-status.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@ containers-storage-status - Output status information from the storage library's
## DESCRIPTION
Queries the storage library's driver for status information.

The status output includes a **Supports reflinks** field which indicates whether the filesystem supports reflink/CoW (copy-on-write) file cloning. Reflinks are supported on Btrfs, XFS (with reflink=1 mount option), and OCFS2. When reflinks are supported, storage deduplication will work efficiently.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The status field is targeted at human readers and is not an appropriate mechanism for API consumers. (E.g., purely hypothetically, it might be localized!)

(Also, well, the containers-storage tool is rarely given to users as a supported binary.)

Throughout the Go codebase, this needs to be an explicit API (DedupPossibleInPrinciple?) which returns a boolean. And then we can talk about whether / how it is consumable from shell scripts.


## EXAMPLE

containers-storage status

## SEE ALSO
containers-storage-version(1)
containers-storage-version(1), containers-storage-dedup(1)
18 changes: 18 additions & 0 deletions storage/drivers/overlay/overlay.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ type Driver struct {
stagingDirsLocks map[string]*staging_lockfile.StagingLockFile

supportsIDMappedMounts *bool
supportsReflinks *bool
}

type additionalLayerStore struct {
Expand Down Expand Up @@ -302,6 +303,18 @@ func (d *Driver) getSupportsDataOnly() (bool, error) {
return supportsDataOnly, nil
}

func (d *Driver) getSupportsReflinks() (bool, error) {
if d.supportsReflinks != nil {
return *d.supportsReflinks, nil
}
supportsReflinks, err := checkSupportsReflinks(d.home, d.runhome)
if err != nil {
return false, err
}
d.supportsReflinks = &supportsReflinks
Comment on lines +307 to +314

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

unless I am missing something this is being called in Status without the store lock held which means these field assignments are not concurrent safe and can cause a panic if status is being called concurrently for the first time.

return supportsReflinks, nil
}

// isNetworkFileSystem checks if the specified file system is supported by native overlay
// as backing store when running in a user namespace.
func isNetworkFileSystem(fsMagic graphdriver.FsMagic) bool {
Expand Down Expand Up @@ -866,13 +879,18 @@ func (d *Driver) Status() [][2]string {
if err != nil {
supportsVolatile = false
}
supportsReflinks, err := d.getSupportsReflinks()
if err != nil {
supportsReflinks = false
}
return [][2]string{
{"Backing Filesystem", backingFs},
{"Supports d_type", strconv.FormatBool(d.supportsDType)},
{"Native Overlay Diff", strconv.FormatBool(!d.useNaiveDiff())},
{"Using metacopy", strconv.FormatBool(d.usingMetacopy)},
{"Supports shifting", strconv.FormatBool(d.SupportsShifting(nil, nil))},
{"Supports volatile", strconv.FormatBool(supportsVolatile)},
{"Supports reflinks", strconv.FormatBool(supportsReflinks)},
}
}

Expand Down
89 changes: 89 additions & 0 deletions storage/drivers/overlay/reflink_check_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
//go:build linux

package overlay

import (
"fmt"
"os"
"path/filepath"

"github.com/sirupsen/logrus"
"go.podman.io/storage/pkg/fileutils"
)

func checkSupportsReflinks(home, runhome string) (bool, error) {
const feature = "reflinks"

reflinkCacheResult, _, err := cachedFeatureCheck(runhome, feature)
if err == nil {
logrus.Debugf("Cached reflink support: %v", reflinkCacheResult)
return reflinkCacheResult, nil
}

supportsReflinks, err := testReflinkSupport(home)
if err != nil {
logrus.Debugf("overlay: reflink test failed: %v", err)
supportsReflinks = false
}
logrus.Debugf("overlay: reflink support: %v", supportsReflinks)

if err := cachedFeatureRecord(runhome, feature, supportsReflinks, ""); err != nil {
return false, fmt.Errorf("recording reflink-support status: %w", err)
}

return supportsReflinks, nil
}

func testReflinkSupport(testDir string) (bool, error) {
srcPath := filepath.Join(testDir, ".reflink-test-src")
dstPath := filepath.Join(testDir, ".reflink-test-dst")
defer os.Remove(srcPath)
defer os.Remove(dstPath)

srcFile, err := os.Create(srcPath)
if err != nil {
return false, fmt.Errorf("failed to create source file: %w", err)
}

testData := []byte("reflink test")
if _, err := srcFile.Write(testData); err != nil {
srcFile.Close()
return false, fmt.Errorf("failed to write test data: %w", err)
}

if err := srcFile.Sync(); err != nil {
srcFile.Close()
return false, fmt.Errorf("failed to sync source file: %w", err)
}
srcFile.Close()

srcFile, err = os.Open(srcPath)
if err != nil {
return false, fmt.Errorf("failed to reopen source file: %w", err)
}
defer srcFile.Close()

dstFile, err := os.Create(dstPath)
if err != nil {
return false, fmt.Errorf("failed to create destination file: %w", err)
}
defer dstFile.Close()

err = fileutils.Reflink(srcFile, dstFile)
if err != nil {
logrus.Debugf("reflink test failed: %v", err)
return false, nil
}

dstFile.Close()
dstContent, err := os.ReadFile(dstPath)
if err != nil {
return false, fmt.Errorf("failed to read destination file: %w", err)
}

if string(dstContent) != string(testData) {
return false, fmt.Errorf("reflink data mismatch")
}

return true, nil
}
58 changes: 58 additions & 0 deletions storage/drivers/overlay/reflink_check_linux_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
//go:build linux

package overlay

import (
"os"
"path/filepath"
"testing"
)

func TestCheckSupportsReflinks(t *testing.T) {
tmpHome := t.TempDir()
tmpRunHome := t.TempDir()

supported, err := checkSupportsReflinks(tmpHome, tmpRunHome)
if err != nil {
t.Fatalf("checkSupportsReflinks failed: %v", err)
}

cachedSupported, _, err := cachedFeatureCheck(tmpRunHome, "reflinks")
if err != nil {
t.Fatalf("cachedFeatureCheck failed: %v", err)
}

if supported != cachedSupported {
t.Errorf("cached result mismatch: got %v, want %v", cachedSupported, supported)
}

supported2, err := checkSupportsReflinks(tmpHome, tmpRunHome)
if err != nil {
t.Fatalf("checkSupportsReflinks second call failed: %v", err)
}

if supported != supported2 {
t.Errorf("second call result mismatch: got %v, want %v", supported2, supported)
}
}

func TestTestReflinkSupport(t *testing.T) {
tmpDir := t.TempDir()

supported, err := testReflinkSupport(tmpDir)
if err != nil {
t.Logf("testReflinkSupport error (may be expected on non-reflink filesystem): %v", err)
}

srcPath := filepath.Join(tmpDir, ".reflink-test-src")
dstPath := filepath.Join(tmpDir, ".reflink-test-dst")

if _, err := os.Stat(srcPath); !os.IsNotExist(err) {
t.Errorf("source file was not cleaned up: %v", err)
}
if _, err := os.Stat(dstPath); !os.IsNotExist(err) {
t.Errorf("destination file was not cleaned up: %v", err)
}

t.Logf("Reflink support on this filesystem: %v", supported)
}