From 68dc7774642ae618e06baa55a65e688eca3086dc Mon Sep 17 00:00:00 2001 From: Jun Lin Chen Date: Sun, 20 Sep 2020 14:40:25 -0400 Subject: [PATCH 1/6] stargz: regFileEntry might be nil --- stargz/stargz.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stargz/stargz.go b/stargz/stargz.go index a01876a..df7dcff 100644 --- a/stargz/stargz.go +++ b/stargz/stargz.go @@ -785,7 +785,7 @@ func (w *Writer) AppendTar(r io.Reader) error { } else { w.toc.Entries = append(w.toc.Entries, ent) } - if payloadDigest != nil { + if regFileEntry != nil && payloadDigest != nil { regFileEntry.Digest = fmt.Sprintf("sha256:%x", payloadDigest.Sum(nil)) } if err := tw.Flush(); err != nil { From 4a2a6bdd4ef871cdbf4246a298f1d52e42e7964a Mon Sep 17 00:00:00 2001 From: Jun Lin Chen Date: Mon, 30 Nov 2020 16:29:05 -0500 Subject: [PATCH 2/6] ADD: attribute for file size after compression. --- stargz/stargz.go | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/stargz/stargz.go b/stargz/stargz.go index df7dcff..d533782 100644 --- a/stargz/stargz.go +++ b/stargz/stargz.go @@ -187,6 +187,8 @@ type TOCEntry struct { ChunkOffset int64 `json:"chunkOffset,omitempty"` ChunkSize int64 `json:"chunkSize,omitempty"` + CompressedSize int64 `json:"compressedSize,omitempty"` + children map[string]*TOCEntry } @@ -573,7 +575,7 @@ func NewWriter(w io.Writer) *Writer { return &Writer{ bw: bw, cw: cw, - toc: &jtoc{Version: 1}, + toc: &jtoc{Version: 2}, diffHash: sha256.New(), } } @@ -755,11 +757,16 @@ func (w *Writer) AppendTar(r io.Reader) error { var written int64 totalSize := ent.Size // save it before we destroy ent tee := io.TeeReader(tr, payloadDigest) + didWrite := false + var prevEnt *TOCEntry + prevEnt = nil for written < totalSize { if err := w.closeGz(); err != nil { return err } - + if prevEnt != nil { + prevEnt.CompressedSize = w.cw.n - prevEnt.Offset + } chunkSize := int64(w.chunkSize()) remain := totalSize - written if remain < chunkSize { @@ -771,17 +778,28 @@ func (w *Writer) AppendTar(r io.Reader) error { ent.ChunkOffset = written w.condOpenGz() - + didWrite = true if _, err := io.CopyN(tw, tee, chunkSize); err != nil { return fmt.Errorf("error copying %q: %v", h.Name, err) } + prevEnt = ent w.toc.Entries = append(w.toc.Entries, ent) written += chunkSize + ent = &TOCEntry{ Name: h.Name, Type: "chunk", } } + if didWrite { + if err := w.closeGz(); err != nil { + return err + } + if prevEnt != nil { + prevEnt.CompressedSize = w.cw.n - prevEnt.Offset + } + w.condOpenGz() + } } else { w.toc.Entries = append(w.toc.Entries, ent) } @@ -791,6 +809,7 @@ func (w *Writer) AppendTar(r io.Reader) error { if err := tw.Flush(); err != nil { return err } + } return nil } From cee0ab0f2132ea34abc9f730bdb438b1eb88532a Mon Sep 17 00:00:00 2001 From: Jun Lin Chen Date: Sun, 6 Dec 2020 23:02:18 -0500 Subject: [PATCH 3/6] Description --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index f422467..61f1323 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,15 @@ Discussion: https://github.com/golang/go/issues/30829 container image, served directly from a container registry (such as [gcr.io](https://gcr.io/)), without pulling it all locally first. + +## Update + +This version of `stargz` is slightly different from Google's `stargz`. +The origin format may case `Gzip(TarF(file1) + TarHeader(file2)) + GZip(TarF(file2) + TarFooter)`. +This schema changes it to `Gzip(Tar(file1)) + Gzip(TarHeader(file2)) + GZip(TarF(file2)) + GZip(TarFooter)`. + + + ## Background Starting a container should be fast. Currently, however, starting a From 9bf8ab811f12d230f5c3852de04cee10336a625c Mon Sep 17 00:00:00 2001 From: Jun Lin Chen Date: Tue, 8 Dec 2020 10:24:36 -0500 Subject: [PATCH 4/6] change package path --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f34043d..7404299 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/google/crfs +module github.com/mc256/crfs go 1.12 From 32f2c52f21794f3d70a6b5bfe26d68facf6b4bef Mon Sep 17 00:00:00 2001 From: Jun Lin Chen Date: Tue, 8 Dec 2020 11:07:01 -0500 Subject: [PATCH 5/6] change dependency --- crfs.go | 2 +- stargz/stargzify/stargzify.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crfs.go b/crfs.go index a5f5953..c2be226 100644 --- a/crfs.go +++ b/crfs.go @@ -42,9 +42,9 @@ import ( "bazil.org/fuse" fspkg "bazil.org/fuse/fs" "cloud.google.com/go/compute/metadata" - "github.com/google/crfs/stargz" namepkg "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/google" + "github.com/mc256/crfs/stargz" "golang.org/x/sys/unix" ) diff --git a/stargz/stargzify/stargzify.go b/stargz/stargzify/stargzify.go index 507591c..7622cbe 100644 --- a/stargz/stargzify/stargzify.go +++ b/stargz/stargzify/stargzify.go @@ -21,7 +21,6 @@ import ( "os" "strings" - "github.com/google/crfs/stargz" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/logs" "github.com/google/go-containerregistry/pkg/name" @@ -31,6 +30,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/mc256/crfs/stargz" ) var ( From bd9c0d21c273b99c0eff03c65311cd3ae64ea522 Mon Sep 17 00:00:00 2001 From: Jun Lin Chen Date: Tue, 8 Dec 2020 11:14:50 -0500 Subject: [PATCH 6/6] change dependency --- crfs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crfs_test.go b/crfs_test.go index 19ca680..1675ead 100644 --- a/crfs_test.go +++ b/crfs_test.go @@ -19,7 +19,7 @@ import ( "bazil.org/fuse" fspkg "bazil.org/fuse/fs" - "github.com/google/crfs/stargz" + "github.com/mc256/crfs/stargz" "golang.org/x/sys/unix" )