Merge branch 'main' into cookiejar

Author: dinosaursrarr

.github/workflows/main.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: "1.21"
+          go-version: "1.22.2"
 
       - name: Install buildifier
         run: make install-buildifier
@@ -36,7 +36,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v4
         with:
-          go-version: "1.21"
+          go-version: "1.22.2"
 
       - name: Install Node
         uses: actions/setup-node@v3
@@ -158,7 +158,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v4
         with:
-          go-version: "1.21"
+          go-version: "1.22.2"
 
       - name: Fetch Release Artifacts
         uses: actions/download-artifact@v3

.github/workflows/pull-request.yml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: "1.21"
+          go-version: "1.22.2"
 
       - name: Install buildifier
         run: make install-buildifier
@@ -34,7 +34,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v4
         with:
-          go-version: "1.21"
+          go-version: "1.22.2"
 
       - name: Install Node
         uses: actions/setup-node@v3

.gitignore

@@ -2,8 +2,8 @@
 .DS_Store
 
 # Rendered Apps
-examples/*.webp
-examples/*.gif
+examples/**/*.webp
+examples/**/*.gif
 
 # Pixlet Binary
 pixlet

README.md

@@ -60,7 +60,7 @@ def main():
 Render and serve it with:
 
 ```console
-curl https://raw.githubusercontent.com/tidbyt/pixlet/main/examples/hello_world.star | \
+curl https://raw.githubusercontent.com/tidbyt/pixlet/main/examples/hello_world/hello_world.star | \
   pixlet serve /dev/stdin
 ```
 
@@ -135,7 +135,7 @@ to show the Bitcoin tracker on your Tidbyt:
 
 ```console
 # render the bitcoin example
-pixlet render examples/bitcoin.star
+pixlet render examples/bitcoin/bitcoin.star
 
 # login to your Tidbyt account
 pixlet login
@@ -144,7 +144,7 @@ pixlet login
 pixlet devices
 
 # push to your favorite Tidbyt
-pixlet push <YOUR DEVICE ID> examples/bitcoin.webp
+pixlet push <YOUR DEVICE ID> examples/bitcoin/bitcoin.webp
 ```
 
 To get the ID for a device, run `pixlet devices`. Alternatively, you can
@@ -158,8 +158,8 @@ If all goes well, you should see the Bitcoin tracker appear on your Tidbyt:
 Pushing an applet to your Tidbyt without an installation ID simply displays your applet one time. If you would like your applet to continously display as part of the rotation, add an installation ID to the push command:
 
 ```console
-pixlet render examples/bitcoin.star
-pixlet push --installation-id <INSTALLATION ID> <YOUR DEVICE ID> examples/bitcoin.webp
+pixlet render examples/bitcoin/bitcoin.star
+pixlet push --installation-id <INSTALLATION ID> <YOUR DEVICE ID> examples/bitcoin/bitcoin.webp
 ```
 
 For example, if we set the `installationID` to "Bitcoin", it would appear in the mobile app as follows:

bundle/bundle.go

@@ -7,10 +7,14 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 
+	"github.com/nlepage/go-tarfs"
+
 	"tidbyt.dev/pixlet/manifest"
+	"tidbyt.dev/pixlet/runtime"
 )
 
 const (
@@ -26,16 +30,12 @@ const (
 
 // AppBundle represents the unpacked bundle in our system.
 type AppBundle struct {
-	Source   []byte
 	Manifest *manifest.Manifest
+	Source   fs.FS
 }
 
-// InitFromPath translates a directory containing an app manifest and source
-// into an AppBundle.
-func InitFromPath(dir string) (*AppBundle, error) {
-	// Load manifest
-	path := filepath.Join(dir, manifest.ManifestFileName)
-	m, err := os.Open(path)
+func FromFS(fs fs.FS) (*AppBundle, error) {
+	m, err := fs.Open(manifest.ManifestFileName)
 	if err != nil {
 		return nil, fmt.Errorf("could not open manifest: %w", err)
 	}
@@ -46,80 +46,39 @@ func InitFromPath(dir string) (*AppBundle, error) {
 		return nil, fmt.Errorf("could not load manifest: %w", err)
 	}
 
-	// Load source
-	path = filepath.Join(dir, man.FileName)
-	s, err := os.Open(path)
-	if err != nil {
-		return nil, fmt.Errorf("could not open app source: %w", err)
-	}
-	defer s.Close()
-
-	src, err := io.ReadAll(s)
-	if err != nil {
-		return nil, fmt.Errorf("could not read app source: %w", err)
-	}
-
 	// Create app bundle struct
 	return &AppBundle{
 		Manifest: man,
-		Source:   src,
+		Source:   fs,
 	}, nil
 }
 
+// FromDir translates a directory containing an app manifest and source
+// into an AppBundle.
+func FromDir(dir string) (*AppBundle, error) {
+	return FromFS(os.DirFS(dir))
+}
+
 // LoadBundle loads a compressed archive into an AppBundle.
 func LoadBundle(in io.Reader) (*AppBundle, error) {
 	gzr, err := gzip.NewReader(in)
 	if err != nil {
-		return nil, fmt.Errorf("could not create gzip reader: %w", err)
+		return nil, fmt.Errorf("creating gzip reader: %w", err)
 	}
 	defer gzr.Close()
 
-	tr := tar.NewReader(gzr)
-	ab := &AppBundle{}
-
-	for {
-		header, err := tr.Next()
+	// read the entire tarball into memory so that we can seek
+	// around it, and so that the underlying reader can be closed.
+	var b bytes.Buffer
+	io.Copy(&b, gzr)
 
-		switch {
-		case err == io.EOF:
-			// If there are no more files in the bundle, validate and return it.
-			if ab.Manifest == nil {
-				return nil, fmt.Errorf("could not find manifest in archive")
-			}
-			if ab.Source == nil {
-				return nil, fmt.Errorf("could not find source in archive")
-			}
-			return ab, nil
-		case err != nil:
-			// If there is an error, return immediately.
-			return nil, fmt.Errorf("could not read archive: %w", err)
-		case header == nil:
-			// If for some reason we end up with a blank header, continue to the
-			// next one.
-			continue
-		case header.Name == AppSourceName:
-			// Load the app source.
-			buff := make([]byte, header.Size)
-			_, err := io.ReadFull(tr, buff)
-			if err != nil {
-				return nil, fmt.Errorf("could not read source from archive: %w", err)
-			}
-			ab.Source = buff
-		case header.Name == manifest.ManifestFileName:
-			// Load the app manifest.
-			buff := make([]byte, header.Size)
-			_, err := io.ReadFull(tr, buff)
-			if err != nil {
-				return nil, fmt.Errorf("could not read manifest from archive: %w", err)
-			}
-
-			man, err := manifest.LoadManifest(bytes.NewReader(buff))
-			if err != nil {
-				return nil, fmt.Errorf("could not load manifest: %w", err)
-			}
-			ab.Manifest = man
-		}
+	r := bytes.NewReader(b.Bytes())
+	fs, err := tarfs.New(r)
+	if err != nil {
+		return nil, fmt.Errorf("creating tarfs: %w", err)
 	}
+
+	return FromFS(fs)
 }
 
 // WriteBundleToPath is a helper to be able to write the bundle to a provided
@@ -137,6 +96,16 @@ func (b *AppBundle) WriteBundleToPath(dir string) error {
 
 // WriteBundle writes a compressed archive to the provided writer.
 func (ab *AppBundle) WriteBundle(out io.Writer) error {
+	// we don't want to naively write the entire source FS to the tarball,
+	// since it could contain a lot of extraneous files. instead, run the
+	// applet and interrogate it for the files it needs to include in the
+	// bundle.
+	app, err := runtime.NewAppletFromFS(ab.Manifest.ID, ab.Source, runtime.WithPrintDisabled())
+	if err != nil {
+		return fmt.Errorf("loading applet for bundling: %w", err)
+	}
+	bundleFiles := app.PathsForBundle()
+
 	// Setup writers.
 	gzw := gzip.NewWriter(out)
 	defer gzw.Close()
@@ -146,7 +115,7 @@ func (ab *AppBundle) WriteBundle(out io.Writer) error {
 
 	// Write manifest.
 	buff := &bytes.Buffer{}
-	err := ab.Manifest.WriteManifest(buff)
+	err = ab.Manifest.WriteManifest(buff)
 	if err != nil {
 		return fmt.Errorf("could not write manifest to buffer: %w", err)
 	}
@@ -166,19 +135,37 @@ func (ab *AppBundle) WriteBundle(out io.Writer) error {
 		return fmt.Errorf("could not write manifest to archive: %w", err)
 	}
 
-	// Write source.
-	hdr = &tar.Header{
-		Name: AppSourceName,
-		Mode: 0600,
-		Size: int64(len(ab.Source)),
-	}
-	err = tw.WriteHeader(hdr)
-	if err != nil {
-		return fmt.Errorf("could not write source header: %w", err)
-	}
-	_, err = tw.Write(ab.Source)
-	if err != nil {
-		return fmt.Errorf("could not write source to archive: %w", err)
+	// write sources.
+	for _, path := range bundleFiles {
+		stat, err := fs.Stat(ab.Source, path)
+		if err != nil {
+			return fmt.Errorf("could not stat %s: %w", path, err)
+		}
+
+		hdr, err := tar.FileInfoHeader(stat, "")
+		if err != nil {
+			return fmt.Errorf("creating header for %s: %w", path, err)
+		}
+		hdr.Name = filepath.ToSlash(path)
+
+		err = tw.WriteHeader(hdr)
+		if err != nil {
+			return fmt.Errorf("writing header for %s: %w", path, err)
+		}
+
+		if !stat.IsDir() {
+			file, err := ab.Source.Open(path)
+			if err != nil {
+				return fmt.Errorf("opening file %s: %w", path, err)
+			}
+
+			written, err := io.Copy(tw, file)
+			if err != nil {
+				return fmt.Errorf("writing file %s: %w", path, err)
+			} else if written != stat.Size() {
+				return fmt.Errorf("did not write entire file %s: %w", path, err)
+			}
+		}
 	}
 
 	return nil

bundle/bundle_test.go

@@ -11,10 +11,10 @@ import (
 
 func TestBundleWriteAndLoad(t *testing.T) {
 	// Ensure we can load the bundle from an app.
-	ab, err := bundle.InitFromPath("testdata/testapp")
+	ab, err := bundle.FromDir("testdata/testapp")
 	assert.NoError(t, err)
 	assert.Equal(t, "test-app", ab.Manifest.ID)
-	assert.True(t, len(ab.Source) > 0)
+	assert.NotNil(t, ab.Source)
 
 	// Create a temp directory.
 	dir, err := os.MkdirTemp("", "")
@@ -32,7 +32,23 @@ func TestBundleWriteAndLoad(t *testing.T) {
 	newBun, err := bundle.LoadBundle(f)
 	assert.NoError(t, err)
 	assert.Equal(t, "test-app", newBun.Manifest.ID)
-	assert.True(t, len(ab.Source) > 0)
+	assert.NotNil(t, ab.Source)
+
+	// Ensure the loaded bundle contains the files we expect.
+	filesExpected := []string{
+		"manifest.yaml",
+		"test_app.star",
+		"test.txt",
+		"a_subdirectory/hi.jpg",
+	}
+	for _, file := range filesExpected {
+		_, err := newBun.Source.Open(file)
+		assert.NoError(t, err)
+	}
+
+	// Ensure the loaded bundle does not contain any extra files.
+	_, err = newBun.Source.Open("unused.txt")
+	assert.ErrorIs(t, err, os.ErrNotExist)
 }
 func TestLoadBundle(t *testing.T) {
 	f, err := os.Open("testdata/bundle.tar.gz")
@@ -41,7 +57,7 @@ func TestLoadBundle(t *testing.T) {
 	ab, err := bundle.LoadBundle(f)
 	assert.NoError(t, err)
 	assert.Equal(t, "test-app", ab.Manifest.ID)
-	assert.True(t, len(ab.Source) > 0)
+	assert.NotNil(t, ab.Source)
 }
 func TestLoadBundleExcessData(t *testing.T) {
 	f, err := os.Open("testdata/excess-files.tar.gz")
@@ -51,5 +67,5 @@ func TestLoadBundleExcessData(t *testing.T) {
 	ab, err := bundle.LoadBundle(f)
 	assert.NoError(t, err)
 	assert.Equal(t, "test-app", ab.Manifest.ID)
-	assert.True(t, len(ab.Source) > 0)
+	assert.NotNil(t, ab.Source)
 }

bundle/testdata/testapp/a_subdirectory/hi.jpg

@@ -4,5 +4,3 @@ name: Test App
 summary: For Testing
 desc: It's an app for testing.
 author: Test Dev
-fileName: test_app.star
-packageName: testapp

bundle/testdata/testapp/manifest.yaml

@@ -5,11 +5,17 @@ Description: It's an app for testing.
 Author: Test Dev
 """
 
+load("a_subdirectory/hi.jpg", hi_jpeg = "file")
 load("render.star", "render")
 load("schema.star", "schema")
+load("test.txt", test_txt = "file")
 
 DEFAULT_WHO = "world"
 
+TEST_TXT_CONTENT = test_txt.readall()
+
+HI_JPEG_BYTES = hi_jpeg.readall("rb")
+
 def main(config):
     who = config.str("who", DEFAULT_WHO)
     message = "Hello, {}!".format(who)

bundle/testdata/testapp/test_app.star

@@ -0,0 +1 @@
+this file is not used in the app