Test e2e workflow with pgs

Author: antoniomika

.env.example

@@ -152,4 +152,20 @@ TUNS_CONSOLE_SECRET=
 GARAGE_CADDYFILE=./caddy/Caddyfile.garage
 GARAGE_METRICS_TOKEN=secret
 GARAGE_ADMIN_TOKEN=secret
-GARAGE_RPC_TOKEN=secret
+
+# Garage requires a random secret for the RPC server. This was generated with:
+# openssl rand -hex 32
+# A different one is used in production.
+GARAGE_RPC_SECRET=69da4443f998314dfda3341ceb77205316a36e2323599f6932fba2c21185c7ef
+
+# For the garage webui
+API_ADMIN_KEY=${GARAGE_ADMIN_TOKEN}
+API_BASE_URL="http://garage:3903"
+S3_ENDPOINT_URL="http://garage:3900"
+
+# Configure the storage type. These are dev only fixtures.
+STORAGE_TYPE=garage
+GARAGE_URL=http://garage:3900
+GARAGE_ADMIN_URL=http://garage:3903
+GARAGE_ROOT_USER=GK03c0bc4880d00a540d929ee9
+GARAGE_ROOT_PASSWORD=88a01d80ba28c04f2c7013445506bdcd883cb44aa32e63731bcd42971697f171

Makefile

@@ -153,15 +153,23 @@ restore:
 
 dev-db-up:
 	$(DOCKER_CMD) compose --profile db up -d
-	sleep 5
+	sleep 2
+	make create migrate
 	make setup-dev-garage
-.PHONY: db-up
+.PHONY: dev-db-up
 
 dev-db-down:
 	$(DOCKER_CMD) compose --profile db down
-.PHONY: db-down
+.PHONY: dev-db-down
+
+setup-dev-db:
+	$(DOCKER_CMD) exec $(GARAGE_CONTAINER) /garage layout assign -z ash -c 10G $(shell $(DOCKER_CMD) exec $(GARAGE_CONTAINER) /garage status 2>&1 | grep -A 1 ID | tail -n1 | awk '{print $$1}')
+	$(DOCKER_CMD) exec $(GARAGE_CONTAINER) /garage layout apply --version 1
+.PHONY: setup-dev-db
 
 setup-dev-garage:
 	$(DOCKER_CMD) exec $(GARAGE_CONTAINER) /garage layout assign -z ash -c 10G $(shell $(DOCKER_CMD) exec $(GARAGE_CONTAINER) /garage status 2>&1 | grep -A 1 ID | tail -n1 | awk '{print $$1}')
 	$(DOCKER_CMD) exec $(GARAGE_CONTAINER) /garage layout apply --version 1
-.PHONY: setup-garage
+	$(DOCKER_CMD) exec $(GARAGE_CONTAINER) /garage key import --yes -n dev GK03c0bc4880d00a540d929ee9 88a01d80ba28c04f2c7013445506bdcd883cb44aa32e63731bcd42971697f171
+	$(DOCKER_CMD) exec $(GARAGE_CONTAINER) /garage key allow --create-bucket dev
+.PHONY: setup-dev-garage

docker-compose.override.yml

@@ -18,6 +18,11 @@ services:
       - "3900-3904:3900-3904"
     tmpfs:
       - /data
+  garage-webui:
+    env_file:
+      - .env.example
+    ports:
+      - "3909:3909"
   imgproxy:
     env_file:
       - .env.example

docker-compose.prod.yml

@@ -53,6 +53,9 @@ services:
       - .env.prod
     volumes:
       - ./data/garage-data:/data
+  garage-webui:
+    env_file:
+      - .env.prod
   pipemgr:
     env_file:
       - .env.prod

docker-compose.yml

@@ -13,7 +13,16 @@ services:
       - db
       - all
   garage:
-    image: dxflrs/garage:v2.0.0-beta1
+    image: dxflrs/garage:v1.1.0
+    restart: always
+    volumes:
+      - ./garage/config.toml:/etc/garage.toml:ro
+    profiles:
+      - db
+      - storage
+      - all
+  garage-webui:
+    image: khairul169/garage-webui:latest
     restart: always
     volumes:
       - ./garage/config.toml:/etc/garage.toml:ro

garage/config.toml

@@ -8,7 +8,7 @@ replication_factor = 1
 rpc_bind_addr = "[::]:3901"
 
 [s3_api]
-s3_region = "ash"
+s3_region = "us-east-1"
 api_bind_addr = "[::]:3900"
 
 [admin]

go.mod

@@ -20,7 +20,7 @@ toolchain go1.24.0
 
 // replace git.sr.ht/~rockorager/vaxis => ../../vaxis
 
-replace git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-golang => github.com/picosh/garage-admin-sdk-golang v0.0.0-20250429033256-41fd1604e078
+replace git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-golang => github.com/picosh/garage-admin-sdk-golang v0.0.0-20250504231927-e10d3d8e125c
 
 require (
 	git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-golang v0.0.0-20250305093803-0806b8639fd3
@@ -282,7 +282,7 @@ require (
 	golang.org/x/image v0.25.0 // indirect
 	golang.org/x/mod v0.24.0 // indirect
 	golang.org/x/net v0.38.0 // indirect
-	golang.org/x/oauth2 v0.28.0 // indirect
+	golang.org/x/oauth2 v0.29.0 // indirect
 	golang.org/x/sync v0.12.0 // indirect
 	golang.org/x/sys v0.31.0 // indirect
 	golang.org/x/term v0.30.0 // indirect
@@ -293,6 +293,7 @@ require (
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 // indirect
 	google.golang.org/grpc v1.71.0 // indirect
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 // indirect
+	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	howett.net/plist v1.0.1 // indirect
 	mvdan.cc/xurls/v2 v2.6.0 // indirect

go.sum

@@ -1,7 +1,6 @@
 package pgs
 
 import (
-	"bytes"
 	"context"
 	"fmt"
 	"io"
@@ -325,12 +324,12 @@ func (h *UploadAssetHandler) Write(s *pssh.SSHServerConnSession, entry *sendutil
 	}
 
 	if entry.Mode.IsDir() {
-		_, _, err := h.Cfg.Storage.PutObject(
+		_, err := h.Cfg.Storage.PutEmptyObject(
 			bucket,
 			path.Join(shared.GetAssetFileName(entry), "._pico_keep_dir"),
-			bytes.NewReader([]byte{}),
 			entry,
 		)
+
 		return "", err
 	}
 
@@ -399,9 +398,14 @@ func (h *UploadAssetHandler) Write(s *pssh.SSHServerConnSession, entry *sendutil
 		sizeRemaining = min(sizeRemaining, specialFileMax)
 	}
 
+	var r io.Reader
+	if entry.Reader != nil {
+		r = utils.NewMaxBytesReader(data.Reader, int64(sizeRemaining))
+	}
+
 	fsize, err := h.writeAsset(
 		s,
-		utils.NewMaxBytesReader(data.Reader, int64(sizeRemaining)),
+		r,
 		data,
 	)
 	if err != nil {
@@ -494,16 +498,17 @@ func (h *UploadAssetHandler) Delete(s *pssh.SSHServerConnSession, entry *senduti
 	})
 
 	if len(sibs) == 0 {
-		_, _, err := h.Cfg.Storage.PutObject(
+		_, err := h.Cfg.Storage.PutEmptyObject(
 			bucket,
 			filepath.Join(pathDir, "._pico_keep_dir"),
-			bytes.NewReader([]byte{}),
 			entry,
 		)
+
 		if err != nil {
 			return err
 		}
 	}
+
 	err = h.Cfg.Storage.DeleteObject(bucket, assetFilepath)
 
 	surrogate := getSurrogateKey(user.Name, projectName)
@@ -551,12 +556,24 @@ func (h *UploadAssetHandler) writeAsset(s *pssh.SSHServerConnSession, reader io.
 		"filename", assetFilepath,
 	)
 
-	_, fsize, err := h.Cfg.Storage.PutObject(
-		data.Bucket,
-		assetFilepath,
-		reader,
-		data.FileEntry,
-	)
+	var fsize int64
+	var err error
+
+	if reader == nil {
+		_, err = h.Cfg.Storage.PutEmptyObject(
+			data.Bucket,
+			assetFilepath,
+			data.FileEntry,
+		)
+	} else {
+		_, fsize, err = h.Cfg.Storage.PutObject(
+			data.Bucket,
+			assetFilepath,
+			reader,
+			data.FileEntry,
+		)
+	}
+
 	return fsize, err
 }
 

pkg/apps/pgs/uploader.go

@@ -145,6 +145,27 @@ func (s *StorageFS) PutObject(bucket Bucket, fpath string, contents io.Reader, e
 	return loc, size, nil
 }
 
+func (s *StorageFS) PutEmptyObject(bucket Bucket, fpath string, entry *utils.FileEntry) (string, error) {
+	loc := filepath.Join(bucket.Path, fpath)
+	err := os.MkdirAll(filepath.Dir(loc), os.ModePerm)
+	if err != nil {
+		return "", err
+	}
+
+	f, err := os.OpenFile(loc, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	if entry.Mtime > 0 {
+		uTime := time.Unix(entry.Mtime, 0)
+		_ = os.Chtimes(loc, uTime, uTime)
+	}
+
+	return loc, nil
+}
+
 func (s *StorageFS) DeleteObject(bucket Bucket, fpath string) error {
 	loc := filepath.Join(bucket.Path, fpath)
 	err := os.Remove(loc)

pkg/pobj/storage/fs.go

@@ -23,6 +23,7 @@ import (
 
 type StorageGarage struct {
 	Client      *minio.Client
+	ClientKey   string
 	Admin       *garage.APIClient
 	AdminCtx    context.Context
 	BucketCache *expirable.LRU[string, CachedBucket]
@@ -34,7 +35,7 @@ var (
 	_ ObjectStorage = (*StorageGarage)(nil)
 )
 
-func NewStorageGarage(logger *slog.Logger, address, user, pass, token string) (*StorageGarage, error) {
+func NewStorageGarage(logger *slog.Logger, address, user, pass, adminAddress, token string) (*StorageGarage, error) {
 	endpoint, err := url.Parse(address)
 	if err != nil {
 		return nil, err
@@ -49,20 +50,26 @@ func NewStorageGarage(logger *slog.Logger, address, user, pass, token string) (*
 		return nil, err
 	}
 
+	adminEndpoint, err := url.Parse(adminAddress)
+	if err != nil {
+		return nil, err
+	}
+
 	configuration := garage.NewConfiguration()
-	configuration.Host = endpoint.Host
-	configuration.Scheme = endpoint.Scheme
+	configuration.Host = adminEndpoint.Host
+	configuration.Scheme = adminEndpoint.Scheme
 
 	client := garage.NewAPIClient(configuration)
 	ctx := context.WithValue(context.Background(), garage.ContextAccessToken, token)
 
-	_, _, err = client.LayoutApi.GetLayout(ctx).Execute()
+	_, _, err = client.LayoutAPI.GetLayout(ctx).Execute()
 	if err != nil {
 		return nil, err
 	}
 
 	mini := &StorageGarage{
 		Client:      mClient,
+		ClientKey:   user,
 		Admin:       client,
 		AdminCtx:    ctx,
 		BucketCache: expirable.NewLRU[string, CachedBucket](2048, nil, cache.CacheTimeout),
@@ -104,18 +111,41 @@ func (s *StorageGarage) UpsertBucket(name string) (Bucket, error) {
 		return bucket, nil
 	}
 
-	err = s.Client.MakeBucket(context.TODO(), name, minio.MakeBucketOptions{})
+	createBucketRequest := garage.NewCreateBucketRequest()
+	createBucketRequest.SetGlobalAlias(name)
+
+	bucketInfo, _, err := s.Admin.BucketAPI.CreateBucket(s.AdminCtx).CreateBucketRequest(*createBucketRequest).Execute()
+	if err != nil {
+		return bucket, err
+	}
+
+	permissions := garage.NewAllowBucketKeyRequestPermissions(true, true, true)
+	allowBucketKeyRequest := garage.NewAllowBucketKeyRequest(bucketInfo.GetId(), s.ClientKey, *permissions)
+
+	_, _, err = s.Admin.BucketAPI.AllowBucketKey(s.AdminCtx).AllowBucketKeyRequest(*allowBucketKeyRequest).Execute()
 	if err != nil {
 		return bucket, err
 	}
 
+	// We can have garage enforce the quota on the bucket
+	// bucketQuotas := garage.NewUpdateBucketRequestQuotas()
+	// bucketQuotas.SetMaxSize()
+
+	// updateBucketRequest := garage.NewUpdateBucketRequest()
+	// updateBucketRequest.SetQuotas(*bucketQuotas)
+
+	// _, _, err = s.Admin.BucketAPI.UpdateBucket(s.AdminCtx).Id(bucketInfo.GetId()).UpdateBucketRequest(*updateBucketRequest).Execute()
+	// if err != nil {
+	// 	return bucket, err
+	// }
+
 	s.BucketCache.Remove(name)
 
 	return bucket, nil
 }
 
 func (s *StorageGarage) GetBucketQuota(bucket Bucket) (uint64, error) {
-	info, _, err := s.Admin.BucketApi.GetBucketInfo(s.AdminCtx).GlobalAlias(bucket.Name).Execute()
+	info, _, err := s.Admin.BucketAPI.GetBucketInfo(s.AdminCtx).GlobalAlias(bucket.Name).Execute()
 	if err != nil {
 		return 0, err
 	}
@@ -124,15 +154,11 @@ func (s *StorageGarage) GetBucketQuota(bucket Bucket) (uint64, error) {
 		return 0, fmt.Errorf("bucket %s not found", bucket.Name)
 	}
 
-	if info.Quotas == nil {
-		return 0, fmt.Errorf("bucket %s has no quota", bucket.Name)
-	}
-
-	if info.Quotas.MaxSize.Get() == nil {
-		return 0, fmt.Errorf("bucket %s has no quota", bucket.Name)
+	if info.Bytes == nil {
+		return 0, fmt.Errorf("bucket %s has no size", bucket.Name)
 	}
 
-	return uint64(*info.Quotas.MaxSize.Get()), nil
+	return uint64(info.GetBytes()), nil
 }
 
 func (s *StorageGarage) ListBuckets() ([]string, error) {
@@ -242,6 +268,26 @@ func (s *StorageGarage) PutObject(bucket Bucket, fpath string, contents io.Reade
 	return fmt.Sprintf("%s/%s", info.Bucket, info.Key), info.Size, nil
 }
 
+func (s *StorageGarage) PutEmptyObject(bucket Bucket, fpath string, entry *utils.FileEntry) (string, error) {
+	opts := minio.PutObjectOptions{
+		UserMetadata: map[string]string{
+			"Mtime": fmt.Sprint(time.Now().Unix()),
+		},
+	}
+
+	if entry.Mtime > 0 {
+		opts.UserMetadata["Mtime"] = fmt.Sprint(entry.Mtime)
+	}
+
+	info, err := s.Client.PutObject(context.TODO(), bucket.Name, fpath, nil, 0, opts)
+
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%s/%s", info.Bucket, info.Key), nil
+}
+
 func (s *StorageGarage) DeleteObject(bucket Bucket, fpath string) error {
 	err := s.Client.RemoveObject(context.TODO(), bucket.Name, fpath, minio.RemoveObjectOptions{})
 	return err

pkg/pobj/storage/garage.go

@@ -112,6 +112,14 @@ func (s *StorageMemory) PutObject(bucket Bucket, fpath string, contents io.Reade
 	return fmt.Sprintf("%s%s", bucket.Path, fpath), int64(len(d)), nil
 }
 
+func (s *StorageMemory) PutEmptyObject(bucket Bucket, fpath string, entry *utils.FileEntry) (string, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.storage[bucket.Path][fpath] = ""
+	return fmt.Sprintf("%s%s", bucket.Path, fpath), nil
+}
+
 func (s *StorageMemory) DeleteObject(bucket Bucket, fpath string) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()