From 150c9de5323544b80a0dbfd7e609b1ea79127661 Mon Sep 17 00:00:00 2001 From: Etai Shuchatowitz Date: Thu, 9 May 2024 17:06:52 -0400 Subject: [PATCH 1/6] is_directory --- proto/viam/app/packages/v1/packages.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/proto/viam/app/packages/v1/packages.proto b/proto/viam/app/packages/v1/packages.proto index ba06265a3..8012859c0 100644 --- a/proto/viam/app/packages/v1/packages.proto +++ b/proto/viam/app/packages/v1/packages.proto @@ -53,6 +53,7 @@ enum PackageType { message FileInfo { string name = 1; uint64 size = 2; + optional bool is_directory = 3; } message PackageInfo { From ce417fcd1e0c8c3b4959151a240796250ba46ba9 Mon Sep 17 00:00:00 2001 From: Etai Shuchatowitz Date: Wed, 25 Jun 2025 12:01:49 -0400 Subject: [PATCH 2/6] poc --- .../service/datamanager/v1/data_manager.proto | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/proto/viam/service/datamanager/v1/data_manager.proto b/proto/viam/service/datamanager/v1/data_manager.proto index f357120ae..9cf96dbc7 100644 --- a/proto/viam/service/datamanager/v1/data_manager.proto +++ b/proto/viam/service/datamanager/v1/data_manager.proto @@ -20,6 +20,11 @@ service DataManagerService { rpc DoCommand(common.v1.DoCommandRequest) returns (common.v1.DoCommandResponse) { option (google.api.http) = {post: "/viam/api/v1/service/datamanager/{name}/do_command"}; } + + // UploadImageToDataset uploads a single image to a given dataset + rpc UploadImageToDataset(UploadImageToDatasetRequest) return (UploadImageToDatasetResponse) { + option (google.api.http) = {post: "/viam/api/v1/service/datamanager/{name}/upload_image_to_dataset"} + } } message SyncRequest { @@ -29,3 +34,13 @@ message SyncRequest { } message SyncResponse {} + +message UploadImageToDatasetRequest { + string name = 1; + string image_uri = 2; + string dataset_id = 3; + // Additional arguments to the method + google.protobuf.Struct extra = 99; +} + +message UploadImageToDatasetResponse {} From 10ee56c297d4d45dac0772bf50bf96631cf2c3ff Mon Sep 17 00:00:00 2001 From: Etai Shuchatowitz Date: Thu, 26 Jun 2025 17:44:29 -0400 Subject: [PATCH 3/6] proto changes --- proto/viam/app/datasync/v1/data_sync.proto | 12 ++++++++++++ proto/viam/service/datamanager/v1/data_manager.proto | 9 +++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/proto/viam/app/datasync/v1/data_sync.proto b/proto/viam/app/datasync/v1/data_sync.proto index b572929b2..1bc4ca14b 100644 --- a/proto/viam/app/datasync/v1/data_sync.proto +++ b/proto/viam/app/datasync/v1/data_sync.proto @@ -25,6 +25,8 @@ service DataSyncService { rpc StreamingDataCaptureUpload(stream viam.app.datasync.v1.StreamingDataCaptureUploadRequest) returns (viam.app.datasync.v1.StreamingDataCaptureUploadResponse) { option (google.api.http) = {post: "/datasync/v1/streaming_data_capture_upload"}; } + // UploadImageToDataset uploads an image and adds it to a datase. + rpc UploadImageToDataset(viam.app.datasync.v1.UploadImageToDatasetRequest) returns (viam.app.datasync.v1.UploadImageToDatasetResponse) {} } // DataCaptureUploadRequest requests to upload the contents and metadata for tabular data. @@ -110,6 +112,7 @@ message UploadMetadata { map method_parameters = 8; string file_extension = 9; repeated string tags = 10; + repeated string dataset_ids = 12; reserved 4, 11; reserved "component_model", "session_id"; @@ -148,3 +151,12 @@ message DataCaptureUploadMetadata { UploadMetadata upload_metadata = 1; SensorMetadata sensor_metadata = 2; } + +// UploadDataToDatasetRequest uploads the data and adds it to a dataset_id. +message UploadImageToDatasetRequest { + UploadMetadata metadata = 1; + FileData file_data = 2; + repeated string dataset_ids = 3; +} + +message UploadImageToDatasetResponse {} diff --git a/proto/viam/service/datamanager/v1/data_manager.proto b/proto/viam/service/datamanager/v1/data_manager.proto index 9cf96dbc7..a13a98792 100644 --- a/proto/viam/service/datamanager/v1/data_manager.proto +++ b/proto/viam/service/datamanager/v1/data_manager.proto @@ -22,8 +22,8 @@ service DataManagerService { } // UploadImageToDataset uploads a single image to a given dataset - rpc UploadImageToDataset(UploadImageToDatasetRequest) return (UploadImageToDatasetResponse) { - option (google.api.http) = {post: "/viam/api/v1/service/datamanager/{name}/upload_image_to_dataset"} + rpc UploadImageToDataset(UploadImageToDatasetRequest) returns (UploadImageToDatasetResponse) { + option (google.api.http) = {post: "/viam/api/v1/service/datamanager/{name}/upload_image_to_dataset"}; } } @@ -37,8 +37,9 @@ message SyncResponse {} message UploadImageToDatasetRequest { string name = 1; - string image_uri = 2; - string dataset_id = 3; + bytes image = 2; + repeated string tags = 3; + repeated string dataset_ids = 4; // Additional arguments to the method google.protobuf.Struct extra = 99; } From a0e757bbc7a5d68fc9360de07e4b3c3bc1e4b63d Mon Sep 17 00:00:00 2001 From: Etai Shuchatowitz Date: Thu, 26 Jun 2025 17:47:06 -0400 Subject: [PATCH 4/6] only change file_upload --- proto/viam/app/datasync/v1/data_sync.proto | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/proto/viam/app/datasync/v1/data_sync.proto b/proto/viam/app/datasync/v1/data_sync.proto index 1bc4ca14b..3f8163aab 100644 --- a/proto/viam/app/datasync/v1/data_sync.proto +++ b/proto/viam/app/datasync/v1/data_sync.proto @@ -25,8 +25,6 @@ service DataSyncService { rpc StreamingDataCaptureUpload(stream viam.app.datasync.v1.StreamingDataCaptureUploadRequest) returns (viam.app.datasync.v1.StreamingDataCaptureUploadResponse) { option (google.api.http) = {post: "/datasync/v1/streaming_data_capture_upload"}; } - // UploadImageToDataset uploads an image and adds it to a datase. - rpc UploadImageToDataset(viam.app.datasync.v1.UploadImageToDatasetRequest) returns (viam.app.datasync.v1.UploadImageToDatasetResponse) {} } // DataCaptureUploadRequest requests to upload the contents and metadata for tabular data. @@ -151,12 +149,3 @@ message DataCaptureUploadMetadata { UploadMetadata upload_metadata = 1; SensorMetadata sensor_metadata = 2; } - -// UploadDataToDatasetRequest uploads the data and adds it to a dataset_id. -message UploadImageToDatasetRequest { - UploadMetadata metadata = 1; - FileData file_data = 2; - repeated string dataset_ids = 3; -} - -message UploadImageToDatasetResponse {} From ab348e3a85592e17e5b714a9ea6c75b3c65aeb46 Mon Sep 17 00:00:00 2001 From: Etai Shuchatowitz Date: Fri, 27 Jun 2025 17:18:38 -0400 Subject: [PATCH 5/6] add mimetype --- proto/viam/service/datamanager/v1/data_manager.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proto/viam/service/datamanager/v1/data_manager.proto b/proto/viam/service/datamanager/v1/data_manager.proto index a13a98792..56f37ff41 100644 --- a/proto/viam/service/datamanager/v1/data_manager.proto +++ b/proto/viam/service/datamanager/v1/data_manager.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package viam.service.datamanager.v1; +import "app/datasync/v1/data_sync.proto"; import "common/v1/common.proto"; import "google/api/annotations.proto"; import "google/protobuf/struct.proto"; @@ -40,6 +41,7 @@ message UploadImageToDatasetRequest { bytes image = 2; repeated string tags = 3; repeated string dataset_ids = 4; + app.datasync.v1.MimeType mime_type = 5; // Additional arguments to the method google.protobuf.Struct extra = 99; } From a2c957ba09798770445f51c273fbbf4d95ca2354 Mon Sep 17 00:00:00 2001 From: Etai Shuchatowitz Date: Fri, 11 Jul 2025 15:57:09 -0400 Subject: [PATCH 6/6] binarydata --- proto/viam/service/datamanager/v1/data_manager.proto | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/proto/viam/service/datamanager/v1/data_manager.proto b/proto/viam/service/datamanager/v1/data_manager.proto index 56f37ff41..9f487469f 100644 --- a/proto/viam/service/datamanager/v1/data_manager.proto +++ b/proto/viam/service/datamanager/v1/data_manager.proto @@ -22,9 +22,9 @@ service DataManagerService { option (google.api.http) = {post: "/viam/api/v1/service/datamanager/{name}/do_command"}; } - // UploadImageToDataset uploads a single image to a given dataset - rpc UploadImageToDataset(UploadImageToDatasetRequest) returns (UploadImageToDatasetResponse) { - option (google.api.http) = {post: "/viam/api/v1/service/datamanager/{name}/upload_image_to_dataset"}; + // UploadBinaryDataToDataset uploads raw bytes to a given dataset + rpc UploadRawDataToDataset(UploadBinaryDataToDatasetRequest) returns (UploadBinaryDataToDatasetResponse) { + option (google.api.http) = {post: "/viam/api/v1/service/datamanager/{name}/upload_binary_data_to_dataset"}; } } @@ -36,7 +36,7 @@ message SyncRequest { message SyncResponse {} -message UploadImageToDatasetRequest { +message UploadBinaryDataToDatasetRequest { string name = 1; bytes image = 2; repeated string tags = 3; @@ -46,4 +46,4 @@ message UploadImageToDatasetRequest { google.protobuf.Struct extra = 99; } -message UploadImageToDatasetResponse {} +message UploadBinaryDataToDatasetResponse {}