From ed010f234483fdc086619845753fd8e4a17b92b3 Mon Sep 17 00:00:00 2001 From: Rich Romanowski <79664656+rromanowski-figure@users.noreply.github.com> Date: Tue, 20 Jan 2026 15:51:26 -0500 Subject: [PATCH 1/4] move to standard cargo config for env variables; update readme with new run commands and other general formatting improvements; add optional health check for storage on startup --- .cargo/config.toml | 24 +++++++++++++ .cargo/fs.config.toml | 3 ++ .cargo/google_cloud.config.toml | 4 +++ README.md | 61 ++++++++++++++++----------------- bin/env | 29 ---------------- src/config.rs | 59 +++++++++++++++++++++---------- src/domain.rs | 2 +- src/lib.rs | 2 +- src/object.rs | 3 +- src/storage/error.rs | 3 ++ src/storage/file_system.rs | 46 ++++++++++++++++++++++--- src/storage/google_cloud.rs | 41 +++++++++++++++++++++- src/storage/mod.rs | 13 +++++-- tests/common/config.rs | 13 ++++--- 14 files changed, 209 insertions(+), 94 deletions(-) create mode 100644 .cargo/fs.config.toml create mode 100644 .cargo/google_cloud.config.toml delete mode 100644 bin/env diff --git a/.cargo/config.toml b/.cargo/config.toml index 8131b16..29610ae 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -2,3 +2,27 @@ rustflags = [ "-Dclippy::suspicious", ] + +[env] +OS_URL="0.0.0.0" +OS_PORT="5000" +DB_CONNECTION_POOL_SIZE="3" +DB_HOST="localhost" +DB_PORT="5432" +DB_USER="postgres" +DB_PASS="password1" +DB_NAME="object-store" +DB_SCHEMA="public" +URI_HOST="example.com" +TRACE_HEADER="x-trace-header" +LOGGING_THRESHOLD_SECONDS="1" +RUST_LOG="warn,object_store=debug" +DD_AGENT_ENABLED="false" +DD_VERSION="undefined" +DD_SERVICE="object-store" +DD_ENV="local" +HEALTH_SERVICE_ENABLED="true" + +# STORAGE +# Check connectivity to storage on startup +STORAGE_HEALTH_CHECK="false" diff --git a/.cargo/fs.config.toml b/.cargo/fs.config.toml new file mode 100644 index 0000000..bef28c2 --- /dev/null +++ b/.cargo/fs.config.toml @@ -0,0 +1,3 @@ +[env] +STORAGE_TYPE="file_system" +STORAGE_BASE_PATH="/tmp" diff --git a/.cargo/google_cloud.config.toml b/.cargo/google_cloud.config.toml new file mode 100644 index 0000000..1e01cd5 --- /dev/null +++ b/.cargo/google_cloud.config.toml @@ -0,0 +1,4 @@ +[env] +STORAGE_TYPE="google_cloud" +# Required - replace/run with actual bucket name +# STORAGE_BASE_PATH="bucket-test" diff --git a/README.md b/README.md index ad0d277..e8d4fee 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,31 @@ # Object Store - -## Status - +An object storage system with strong encryption properties and peer-to-peer replication [![stability-release-candidate](https://img.shields.io/badge/stability-pre--release-48c9b0.svg)](https://github.com/mkenney/software-guides/blob/master/STABILITY-BADGES.md#release-candidate) [![Latest Release][release-badge]][release-latest] [![License][license-badge]][license-url] [release-badge]: https://img.shields.io/github/v/tag/provenance-io/object-store.svg?sort=semver [release-latest]: https://github.com/provenance-io/object-store/releases/latest - [license-badge]: https://img.shields.io/github/license/provenance-io/object-store.svg [license-url]: https://github.com/provenance-io/object-store/blob/main/LICENSE -A object storage system with a gRPC interface and strong encryption properties. Currently, only the -[Provenance DIME](https://developer.provenance.io/docs/pb/p8e/overview/encrypted-object-store/dime-encryption-envelope-specification/) -format is accepted. Supporting material can be found [here](https://developer.provenance.io/docs/pb/p8e/overview/encrypted-object-store/). - -This service can be used directly, but the most common case is to use this alongside the [P8e Execution Environment](https://github.com/provenance-io/p8e-scope-sdk) -in order to process Provenance scopes and memorialize them on chain. - ## Features - -- Strong end-to-end encryption. +- Strong end-to-end encryption - Currently, only the [Provenance DIME](https://developer.provenance.io/docs/pb/p8e/overview/encrypted-object-store/dime-encryption-envelope-specification/) format is accepted. Supporting material can be found [here](https://developer.provenance.io/docs/pb/p8e/overview/encrypted-object-store/). - Peer-to-peer replication to parties (third party `object-store`s) you want to share data with. -- Capable of publishing traces to Datadog. +- Configurable storage backends +- Tracing with [Datadog](https://docs.datadoghq.com). - [Additional features](./docs/FEATURES.md) -## Authentication +## Overview +An object store can be used directly, but the most common case is to use this alongside the [P8e Execution Environment](https://github.com/provenance-io/p8e-scope-sdk) in order to process Provenance scopes and memorialize them on chain. -gRPC metadata based authentication is provided on a per key basis. When adding a public key to the database, an `auth_type` and `auth_data` can be provided. These -can either be leveraged directly as an api key or indirectly be combining it with a proxy capable of authentication and header forwarding. Setting both of these fields -to `null` and a service level config property of `USER_AUTH_ENABLED=false` disables all authentication - this can be used if the object store is meant for internal use -and not exposed publicly. - -- Example authentication configuration +### Authentication +gRPC metadata based authentication is provided on a per key basis. +When adding a public key to the database, an `auth_type` and `auth_data` can be provided. +These can either be leveraged directly as an api key or indirectly be combining it with a proxy capable of authentication and header forwarding. +Setting both of these fields to `null` and a service level config property of `USER_AUTH_ENABLED=false` disables all authentication - this can be used if the object store is meant for internal use and not exposed publicly. +#### Example authentication configuration NOTE: Requires settings the service level configuration to `USER_AUTH_ENABLED=true`. ``` @@ -44,22 +35,30 @@ auth_type=header auth_data=x-custom-header:6eace982-f682-4b1d-9f8e-82ed9ab15813 ``` -With such a configuration all requests for this public key will have to contain this metadata. - -## Backends +With such a configuration, all requests for this public key _must_ contain this metadata. -This service was designed to support many underlying storage backends. The currently supported backends are `postgres`, `google cloud storage`, and the local `file system`. -In practice, a sizeable number of objects this system stores are very small. For this reason the `postgres` backend, along with a byte threshold, is provided -so that items smaller than the threshold specified can have thier bytes stored directly in the database. +### Storage +This system supports different storage backends. Currently native file system and Google Cloud are supported. +In practice, many objects are very small. Therefore the `postgres` backend, along with a byte threshold, is provided where items smaller than the threshold are stored directly in the database. ## Development +The most common use case for `object-store` is to run it alongside `p8e` in order to write to the [Provenance blockchain](https://provenance.io). +The simplest way to get this up and running is [here](https://github.com/provenance-io/p8e-scope-sdk/tree/main/dev-tools/compose). ### Running Locally +A postgres connection and data directory is required to run locally. +A base set of environment variables are defined in [.cargo/config.toml](./.cargo/config.toml). +Then, specify additional config with one of the sibling config files and/or env variables. -A postgres connection and data directory is required to run locally. A base set of environment variables can be edited and sourced from `./bin/env`. - -The most common use case for `object-store` is to run it alongside `p8e` in order to write to the [Provenance blockchain](https://provenance.io). The simplest way to get this -up and running is [here](https://github.com/provenance-io/p8e-scope-sdk/tree/main/dev-tools/compose). +**Run with file system storage** +```shell +cargo --config .cargo/fs.config.toml run +``` +**Run with google cloud storage and health check** +```shell +STORAGE_HEALTH_CHECK=true STORAGE_BASE_PATH=my-custom-bucket \ + cargo --config .cargo/google_cloud.config.toml run +``` ### Contributing It is recommended to have `rust-analyzer` installed and configured for your IDE. diff --git a/bin/env b/bin/env deleted file mode 100644 index 29e399e..0000000 --- a/bin/env +++ /dev/null @@ -1,29 +0,0 @@ -export OS_URL=0.0.0.0 -export OS_PORT=5000 -export DB_CONNECTION_POOL_SIZE=3 -export DB_HOST=localhost -export DB_PORT=5432 -export DB_USER=postgres -export DB_PASS=password1 -export DB_NAME=object-store -export DB_SCHEMA=public -export URI_HOST=figure.com -export TRACE_HEADER=x-trace-header -export LOGGING_THRESHOLD_SECONDS=1 -export RUST_LOG=warn,object_store=debug -export DD_AGENT_ENABLED=false -export DD_VERSION=undefined -export DD_SERVICE=object-store -export DD_ENV=local -export HEALTH_SERVICE_ENABLED=true - -## start of file system variables -export STORAGE_TYPE=file_system -export STORAGE_BASE_PATH=storage_bucket -## end - -## start of google cloud variables -# export STORAGE_TYPE=google_cloud -# export STORAGE_BASE_PATH=scirner-os-test -# export SERVICE_ACCOUNT=key-file -## end diff --git a/src/config.rs b/src/config.rs index 1f33606..8b91748 100644 --- a/src/config.rs +++ b/src/config.rs @@ -26,6 +26,43 @@ pub struct ReplicationConfig { pub snapshot_cache_refresh_frequency: TimeDelta, } +#[derive(Clone, Debug)] +pub struct StorageConfig { + /// One of: `file_system`, `google_cloud` + pub storage_type: String, + pub storage_base_url: Option, + pub storage_base_path: String, + /// Objects with size, in bytes, below this threshold will be stored in database. + /// Larger objects will be in configured storage + pub storage_threshold: usize, + pub health_check: bool, +} + +impl StorageConfig { + pub fn from_env() -> Self { + let storage_type = env::var("STORAGE_TYPE").expect("STORAGE_TYPE not set"); + let storage_base_url = env::var("STORAGE_BASE_URL").ok(); + let storage_base_path = env::var("STORAGE_BASE_PATH").expect("STORAGE_BASE_PATH not set"); + let storage_threshold = env::var("STORAGE_THRESHOLD") + // default to ~ 5KB + .unwrap_or("5000".to_owned()) + .parse() + .expect("STORAGE_THRESHOLD could not be parsed into a u32"); + let health_check = env::var("STORAGE_HEALTH_CHECK") + .unwrap_or("false".to_owned()) + .parse() + .expect("STORAGE_HEALTH_CHECK could not be parsed into a bool"); + + Self { + storage_type, + storage_base_url, + storage_base_path, + storage_threshold, + health_check, + } + } +} + impl ReplicationConfig { pub fn new( replication_enabled: bool, @@ -57,13 +94,7 @@ pub struct Config { pub db_password: String, pub db_database: String, pub db_schema: String, - /// One of: `file_system`, `google_cloud` - pub storage_type: String, - pub storage_base_url: Option, - pub storage_base_path: String, - /// Objects with size, in bytes, below this threshold will be stored in database. - /// Larger objects will be in configured storage - pub storage_threshold: usize, + pub storage_config: StorageConfig, pub replication_config: ReplicationConfig, /// If None, trace middleware [MinitraceGrpcMiddlewareLayer][crate::middleware::MinitraceGrpcMiddlewareLayer] disabled pub dd_config: Option, @@ -105,14 +136,7 @@ impl Config { let db_password = env::var("DB_PASS").expect("DB_PASS not set"); let db_database = env::var("DB_NAME").expect("DB_NAME not set"); let db_schema = env::var("DB_SCHEMA").expect("DB_SCHEMA not set"); - let storage_type = env::var("STORAGE_TYPE").expect("STORAGE_TYPE not set"); - let storage_base_url = env::var("STORAGE_BASE_URL").ok(); - let storage_base_path = env::var("STORAGE_BASE_PATH").expect("STORAGE_BASE_PATH not set"); - let storage_threshold = env::var("STORAGE_THRESHOLD") - // default to ~ 5KB - .unwrap_or("5000".to_owned()) - .parse() - .expect("STORAGE_THRESHOLD could not be parsed into a u32"); + let replication_batch_size = env::var("REPLICATION_BATCH_SIZE") .unwrap_or("10".to_owned()) .parse() @@ -221,10 +245,7 @@ impl Config { db_password, db_database, db_schema, - storage_type, - storage_base_url, - storage_base_path, - storage_threshold, + storage_config: StorageConfig::from_env(), replication_config, dd_config, logging_threshold_seconds, diff --git a/src/domain.rs b/src/domain.rs index c03113a..707a03d 100644 --- a/src/domain.rs +++ b/src/domain.rs @@ -33,7 +33,7 @@ impl ObjectApiResponse for Object { }), hash: self.hash.decoded()?, uri: format!("object://{}/{}", &config.uri_host, &self.hash), - bucket: config.storage_base_path.clone(), + bucket: config.storage_config.storage_base_path.clone(), name: self.name.clone(), metadata: Some(ObjectMetadata { sha512: Vec::new(), // TODO get hash of whole dime? diff --git a/src/lib.rs b/src/lib.rs index 2d778ce..833667b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -60,7 +60,7 @@ impl AppContext { pub async fn new(config: Arc) -> Result { let db_pool = connect_and_migrate(&config).await?; let cache = Cache::new(db_pool.clone()).await?; - let storage = new_storage(&config).await?; + let storage = new_storage(&config.storage_config).await?; let admin_service = AdminGrpc::new(config.clone()); let public_key_service = PublicKeyGrpc::new(cache.clone(), config.clone(), db_pool.clone()); diff --git a/src/object.rs b/src/object.rs index d2a19c8..7ee5092 100644 --- a/src/object.rs +++ b/src/object.rs @@ -277,7 +277,8 @@ impl ObjectService for ObjectGrpc { // mail items should be under any reasonable threshold set, but explicitly check for // mail and always used database storage let is_mail = dime.metadata.contains_key(consts::MAILBOX_KEY); - let above_storage_threshold = dime_properties.dime_length > self.config.storage_threshold; + let above_storage_threshold = + dime_properties.dime_length > self.config.storage_config.storage_threshold; let response = if !is_mail && above_storage_threshold { let response = datastore::put_object( diff --git a/src/storage/error.rs b/src/storage/error.rs index 5672792..aec9d5f 100644 --- a/src/storage/error.rs +++ b/src/storage/error.rs @@ -5,6 +5,9 @@ quick_error! { pub enum StorageError { IoError(message: String) { } ContentLengthError(message: String) { } + Utf8Error(err: std::string::FromUtf8Error) { + from() + } } } diff --git a/src/storage/file_system.rs b/src/storage/file_system.rs index 8efd543..e284b25 100644 --- a/src/storage/file_system.rs +++ b/src/storage/file_system.rs @@ -28,7 +28,7 @@ impl FileSystem { Ok(_) => Ok(()), Err(e) => match e.kind() { std::io::ErrorKind::AlreadyExists => Ok(()), - _ => Err(StorageError::IoError(format!("{:?}", e))), + _ => Err(StorageError::IoError(format!("{:?} - {:?}", e, path_buf))), }, } } @@ -36,30 +36,53 @@ impl FileSystem { #[async_trait::async_trait] impl Storage for FileSystem { + #[trace(name = "file_system::health_check")] + async fn health_check(&self) -> Result<()> { + let test_file = StoragePath { + dir: "test".to_owned(), + file: "test.txt".to_owned(), + }; + + self.store(&test_file, 12, b"fs connected").await?; + + let response = self.fetch(&test_file, 12).await?; + + let contents = String::from_utf8(response)?; + + log::debug!("health check - {} contents: {}", test_file, contents); + + // Ignore delete errors + let _ = self.delete(&test_file).await; + + Ok(()) + } + #[trace(name = "file_system::store")] async fn store(&self, path: &StoragePath, content_length: usize, data: &[u8]) -> Result<()> { if let Err(e) = self.validate_content_length(path, content_length, data) { log::warn!("{:?}", e); } + let file_path = self.get_path(path); + let file = OpenOptions::new() .write(true) .create_new(true) - .open(self.get_path(path)) + .open(&file_path) .await; match file { Ok(mut file) => file .write_all(data) .await - .map_err(|e| StorageError::IoError(format!("{:?}", e))), + .map_err(|e| StorageError::IoError(format!("{:?}: {:?}", e, file_path))), Err(e) => match e.kind() { std::io::ErrorKind::AlreadyExists => Ok(()), std::io::ErrorKind::NotFound => { self.create_dir(path).await?; self.store(path, content_length, data).await } - _ => Err(StorageError::IoError(format!("{:?}", e))), + _ => Err(StorageError::IoError(format!("{:?}: {:?}", e, file_path))), }, } } @@ -79,6 +102,21 @@ impl Storage for FileSystem { Ok(data) } + + #[trace(name = "file_system::delete")] + async fn delete(&self, path: &StoragePath) -> Result<()> { + let full_path = self.get_path(path); + let absolute_path = full_path.canonicalize().unwrap_or(full_path.clone()); + + tokio::fs::remove_file(full_path).await.map_err(|e| { + StorageError::IoError(format!( + "Unable to delete file: {:?} {:?}", + absolute_path, e + )) + })?; + + Ok(()) + } } #[cfg(test)] diff --git a/src/storage/google_cloud.rs b/src/storage/google_cloud.rs index 5c791fb..1a1d13f 100644 --- a/src/storage/google_cloud.rs +++ b/src/storage/google_cloud.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; use bytes::Bytes; use fastrace_macro::trace; -use google_cloud_storage::client::Storage as GcsStorage; +use google_cloud_storage::client::{Storage as GcsStorage, StorageControl}; use crate::storage::{Result, Storage, StorageError, StoragePath}; @@ -32,6 +32,27 @@ impl GoogleCloud { #[async_trait::async_trait] impl Storage for GoogleCloud { + #[trace(name = "google_cloud::health_check")] + async fn health_check(&self) -> Result<()> { + let test_file = StoragePath { + dir: "test".to_owned(), + file: "test.txt".to_owned(), + }; + + self.store(&test_file, 15, b"cloud connected").await?; + + let response = self.fetch(&test_file, 15).await?; + + let contents = String::from_utf8(response)?; + + log::debug!("health check - test/test.txt contents: {}", contents); + + // Ignore delete errors + let _ = self.delete(&test_file).await; + + Ok(()) + } + #[trace(name = "google_cloud::store")] async fn store(&self, path: &StoragePath, content_length: usize, data: &[u8]) -> Result<()> { if let Err(e) = self.validate_content_length(path, content_length, data) { @@ -97,4 +118,22 @@ impl Storage for GoogleCloud { Ok(data) } + + #[trace(name = "google_cloud::delete")] + async fn delete(&self, path: &StoragePath) -> Result<()> { + let bucket_path = self.bucket(); + let full_path = self.get_path(path); + + let control_client = StorageControl::builder().build().await.unwrap(); + + // Ignore delete errors + let _response = control_client + .delete_object() + .set_bucket(bucket_path) + .set_object(full_path.to_str().unwrap()) + .send() + .await; + + Ok(()) + } } diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 99db789..86398ac 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -9,7 +9,7 @@ pub use error::*; pub use file_system::FileSystem; pub use google_cloud::GoogleCloud; -use crate::{config::Config, types::OsError}; +use crate::{config::StorageConfig, types::OsError}; // TODO implement checksum in filestore @@ -26,9 +26,12 @@ impl Display for StoragePath { #[async_trait::async_trait] pub trait Storage: Send + Sync + std::fmt::Debug { + async fn health_check(&self) -> Result<()>; + // store should be idempotent async fn store(&self, path: &StoragePath, content_length: usize, data: &[u8]) -> Result<()>; async fn fetch(&self, path: &StoragePath, content_length: usize) -> Result>; + async fn delete(&self, path: &StoragePath) -> Result<()>; fn validate_content_length( &self, @@ -50,7 +53,9 @@ pub trait Storage: Send + Sync + std::fmt::Debug { } // TODO fix await unwrap -pub async fn new_storage(config: &Config) -> core::result::Result>, OsError> { +pub async fn new_storage( + config: &StorageConfig, +) -> core::result::Result>, OsError> { let storage = match config.storage_type.as_str() { "file_system" => Ok(Box::new(FileSystem::new(PathBuf::from( config.storage_base_path.as_str(), @@ -65,5 +70,9 @@ pub async fn new_storage(config: &Config) -> core::result::Result Err(OsError::InvalidApplicationState("".to_owned())), }?; + if config.health_check { + storage.health_check().await?; + } + Ok(Arc::new(storage)) } diff --git a/tests/common/config.rs b/tests/common/config.rs index ec1db8e..15f8735 100644 --- a/tests/common/config.rs +++ b/tests/common/config.rs @@ -1,5 +1,5 @@ use chrono::Duration; -use object_store::config::{Config, DatadogConfig, ReplicationConfig}; +use object_store::config::{Config, DatadogConfig, ReplicationConfig, StorageConfig}; /// Builds a default config suitable for most tests. /// @@ -28,10 +28,13 @@ pub fn test_config(db_port: u16) -> Config { db_password: "postgres".to_owned(), db_database: "postgres".to_owned(), db_schema: "public".to_owned(), - storage_type: "file_system".to_owned(), - storage_base_url: None, - storage_base_path: std::env::temp_dir().to_string_lossy().to_string(), - storage_threshold: 5000, + storage_config: StorageConfig { + storage_type: "file_system".to_owned(), + storage_base_url: None, + storage_base_path: std::env::temp_dir().to_string_lossy().to_string(), + storage_threshold: 5000, + health_check: false, + }, replication_config: ReplicationConfig::new(true, 2, 1, 1, Duration::minutes(5)), dd_config: Some(dd_config), logging_threshold_seconds: 1f64, From c0a653cc7b6a0c32d95c007a2374b9e7314202eb Mon Sep 17 00:00:00 2001 From: Rich Romanowski <79664656+rromanowski-figure@users.noreply.github.com> Date: Tue, 20 Jan 2026 15:58:57 -0500 Subject: [PATCH 2/4] readme newline --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e8d4fee..abf9c2f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Object Store An object storage system with strong encryption properties and peer-to-peer replication + [![stability-release-candidate](https://img.shields.io/badge/stability-pre--release-48c9b0.svg)](https://github.com/mkenney/software-guides/blob/master/STABILITY-BADGES.md#release-candidate) [![Latest Release][release-badge]][release-latest] [![License][license-badge]][license-url] From 5938b14e207ba465fd8f2c7bfb41278280f194b0 Mon Sep 17 00:00:00 2001 From: Rich Romanowski <79664656+rromanowski-figure@users.noreply.github.com> Date: Tue, 20 Jan 2026 17:44:29 -0500 Subject: [PATCH 3/4] add ca-certificates to image --- Cargo.lock | 2 +- Cargo.toml | 2 +- Dockerfile | 12 ++++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ae3495e..5168f79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1920,7 +1920,7 @@ dependencies = [ [[package]] name = "object-store" -version = "1.0.0-alpha.5" +version = "1.0.0-alpha.6" dependencies = [ "async-trait", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index 9756f4f..110efa3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "object-store" -version = "1.0.0-alpha.5" +version = "1.0.0-alpha.6" authors = ["Stephen Cirner "] edition = "2024" diff --git a/Dockerfile b/Dockerfile index bd914ec..83d7fc8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,14 +18,18 @@ FROM debian:bookworm-slim LABEL org.opencontainers.image.source=https://github.com/provenance-io/object-store -RUN apt-get update && apt-get install -y \ - libssl-dev +RUN apt-get update && \ + apt-get install -y ca-certificates libssl3 && \ + rm -rf /var/lib/apt/lists/* EXPOSE 8080 -COPY --from=builder /usr/local/cargo/bin/object-store /usr/local/bin/object-store +COPY --from=builder \ + /usr/local/cargo/bin/object-store /usr/local/bin/object-store + +COPY --from=builder \ + /bin/grpc_health_probe /bin/grpc_health_probe -COPY --from=builder /bin/grpc_health_probe /bin/grpc_health_probe RUN chmod +x /bin/grpc_health_probe CMD ["object-store"] From bf373f8bcefdb3768909b6a4ef6f9c645f9c208d Mon Sep 17 00:00:00 2001 From: Rich Romanowski <79664656+rromanowski-figure@users.noreply.github.com> Date: Tue, 20 Jan 2026 18:07:44 -0500 Subject: [PATCH 4/4] change storage_type to enum --- src/config.rs | 24 ++++++++++++++++++++++-- src/storage/mod.rs | 22 +++++++++++++--------- tests/common/config.rs | 4 ++-- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/config.rs b/src/config.rs index 8b91748..cd3c07f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,6 @@ use std::env; use std::net::{IpAddr, SocketAddr}; +use std::str::FromStr; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; @@ -26,10 +27,27 @@ pub struct ReplicationConfig { pub snapshot_cache_refresh_frequency: TimeDelta, } +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum StorageType { + FileSystem = 0, + GoogleCloud = 1, +} + +impl FromStr for StorageType { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "file_system" => Ok(StorageType::FileSystem), + "google_cloud" => Ok(StorageType::GoogleCloud), + _ => Err(format!("Invalid storage: {}", s)), + } + } +} + #[derive(Clone, Debug)] pub struct StorageConfig { - /// One of: `file_system`, `google_cloud` - pub storage_type: String, + pub storage_type: StorageType, pub storage_base_url: Option, pub storage_base_path: String, /// Objects with size, in bytes, below this threshold will be stored in database. @@ -41,6 +59,8 @@ pub struct StorageConfig { impl StorageConfig { pub fn from_env() -> Self { let storage_type = env::var("STORAGE_TYPE").expect("STORAGE_TYPE not set"); + let storage_type = StorageType::from_str(&storage_type).expect("STORAGE_TYPE invalid"); + let storage_base_url = env::var("STORAGE_BASE_URL").ok(); let storage_base_path = env::var("STORAGE_BASE_PATH").expect("STORAGE_BASE_PATH not set"); let storage_threshold = env::var("STORAGE_THRESHOLD") diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 86398ac..054e43e 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -9,7 +9,10 @@ pub use error::*; pub use file_system::FileSystem; pub use google_cloud::GoogleCloud; -use crate::{config::StorageConfig, types::OsError}; +use crate::{ + config::{StorageConfig, StorageType}, + types::OsError, +}; // TODO implement checksum in filestore @@ -56,19 +59,20 @@ pub trait Storage: Send + Sync + std::fmt::Debug { pub async fn new_storage( config: &StorageConfig, ) -> core::result::Result>, OsError> { - let storage = match config.storage_type.as_str() { - "file_system" => Ok(Box::new(FileSystem::new(PathBuf::from( - config.storage_base_path.as_str(), - ))) as Box), - "google_cloud" => { + let storage = match config.storage_type { + StorageType::FileSystem => { + let file_system = FileSystem::new(PathBuf::from(config.storage_base_path.as_str())); + + Box::new(file_system) as Box + } + StorageType::GoogleCloud => { let google_cloud = GoogleCloud::new(config.storage_base_path.clone()) .await .unwrap(); - Ok(Box::new(google_cloud) as Box) + Box::new(google_cloud) as Box } - _ => Err(OsError::InvalidApplicationState("".to_owned())), - }?; + }; if config.health_check { storage.health_check().await?; diff --git a/tests/common/config.rs b/tests/common/config.rs index 15f8735..73e55a5 100644 --- a/tests/common/config.rs +++ b/tests/common/config.rs @@ -1,5 +1,5 @@ use chrono::Duration; -use object_store::config::{Config, DatadogConfig, ReplicationConfig, StorageConfig}; +use object_store::config::{Config, DatadogConfig, ReplicationConfig, StorageConfig, StorageType}; /// Builds a default config suitable for most tests. /// @@ -29,7 +29,7 @@ pub fn test_config(db_port: u16) -> Config { db_database: "postgres".to_owned(), db_schema: "public".to_owned(), storage_config: StorageConfig { - storage_type: "file_system".to_owned(), + storage_type: StorageType::FileSystem, storage_base_url: None, storage_base_path: std::env::temp_dir().to_string_lossy().to_string(), storage_threshold: 5000,