diff --git a/contracts/sbt/src/compression.rs b/contracts/sbt/src/compression.rs index 8cf6b99..a411124 100644 --- a/contracts/sbt/src/compression.rs +++ b/contracts/sbt/src/compression.rs @@ -1,254 +1,296 @@ -//! SBT metadata compression and decompression using delta + RLE encoding. +//! Credential metadata compression using a MessagePack extension envelope. //! -//! This module provides efficient compression for SBT metadata that leverages -//! the structured nature of JSON metadata while maintaining backward compatibility. -//! -//! ## Compression Strategy -//! -//! 1. **Delta Encoding**: Store differences between consecutive bytes rather than -//! absolute values. Common metadata patterns (repeated fields, similar values) -//! compress well with this technique. -//! -//! 2. **Run-Length Encoding (RLE)**: Compress runs of identical bytes after delta -//! encoding. Typical for padding, repeated quotes, or structural elements. -//! -//! 3. **Magic Prefix**: Compressed data starts with `0xC1` to distinguish from -//! uncompressed metadata (which won't start with control bytes). -//! -//! ## Format -//! -//! ```text -//! Compressed: -//! [MAGIC: 0xC1] -//! [is_delta_encoded: u8] (0 or 1) -//! ([count: u8][value: i8])* if delta-encoded -//! or -//! ([byte: u8])* if not delta-encoded -//! ``` +//! The extension payload uses a small PackBits-style encoding and selects +//! between direct bytes and delta bytes. Inputs that would not become smaller +//! are returned unchanged, preserving compatibility with existing metadata. use soroban_sdk::{contracterror, Bytes, Env}; -/// First byte marking compressed SBT metadata. -const COMPRESSION_MAGIC: u8 = 0xC1; +/// Maximum supported uncompressed credential metadata size. +pub const MAX_METADATA_SIZE: u32 = 4096; + +const MESSAGEPACK_EXT8: u8 = 0xC7; +const MESSAGEPACK_EXT16: u8 = 0xC8; +const ETHOS_METADATA_TYPE: u8 = 0x45; +const FORMAT_VERSION: u8 = 1; +const MODE_DIRECT: u8 = 0; +const MODE_DELTA: u8 = 1; +const LEGACY_MAGIC: u8 = 0xC1; +const MAX_LITERAL_LENGTH: u32 = 128; +const MAX_REPEAT_LENGTH: u32 = 130; #[contracterror] #[derive(Copy, Clone, Debug, Eq, PartialEq)] #[repr(u32)] pub enum CompressionError { - /// Input metadata was empty. - EmptyMetadata = 200, - /// Compressed data is malformed. - InvalidCompressedData = 201, - /// Decompressed output would exceed maximum size. - OutputTooLarge = 202, + /// The MessagePack extension or its block stream is malformed. + InvalidCompressedData = 200, + /// Decompressed output would exceed the metadata size limit. + OutputTooLarge = 201, } -/// Compress metadata using delta + RLE encoding. -/// -/// Returns `Ok(Bytes)` with compressed data if successful, or an error if the -/// input is empty or would fail to compress. -/// -/// # Arguments +/// Compress metadata into the Ethos MessagePack extension format. /// -/// * `env` - Soroban environment -/// * `metadata` - Uncompressed metadata bytes -/// * `max_compressed_size` - Maximum allowed output size (typically 4096) -pub fn compress_metadata( - env: &Env, - metadata: &Bytes, - max_compressed_size: u32, -) -> Result { - if metadata.is_empty() { - return Err(CompressionError::EmptyMetadata); +/// The original bytes are returned when the input is empty, exceeds the +/// metadata limit, or would not become smaller after framing. +pub fn compress_metadata(env: &Env, metadata: &Bytes) -> Bytes { + if metadata.is_empty() || metadata.len() > MAX_METADATA_SIZE { + return metadata.clone(); } - // Try delta encoding for structured data (JSON is highly compressible this way) - if let Ok(delta_compressed) = compress_with_delta(env, metadata, max_compressed_size) { - return Ok(delta_compressed); - } + let direct = encode_blocks(env, metadata, MODE_DIRECT); + let delta = encode_blocks(env, metadata, MODE_DELTA); + let payload = if delta.len() < direct.len() { + delta + } else { + direct + }; + let framed = frame_messagepack_extension(env, &payload); - // Fallback to simple RLE if delta encoding doesn't help - compress_with_rle(env, metadata, max_compressed_size) + if framed.len() < metadata.len() { + framed + } else { + metadata.clone() + } } -/// Decompress metadata that was compressed with `compress_metadata`. -/// -/// # Arguments +/// Decompress MessagePack-framed, legacy, or uncompressed metadata. /// -/// * `env` - Soroban environment -/// * `compressed` - Compressed metadata bytes (should start with COMPRESSION_MAGIC) -/// * `max_decompressed_size` - Maximum allowed output size (typically 4096) +/// Ordinary uncompressed bytes are returned unchanged. The legacy `0xC1` +/// delta/RLE format remains readable for backwards compatibility. pub fn decompress_metadata( env: &Env, - compressed: &Bytes, - max_decompressed_size: u32, + metadata: &Bytes, ) -> Result { - if compressed.is_empty() { - return Err(CompressionError::InvalidCompressedData); + if let Some((payload_start, payload_end, mode)) = parse_messagepack_header(metadata)? { + return decode_blocks(env, metadata, payload_start, payload_end, mode); } - let magic = compressed.get(0).unwrap(); - if magic != COMPRESSION_MAGIC { - return Err(CompressionError::InvalidCompressedData); + if is_legacy_compressed(metadata) { + return decompress_legacy(env, metadata); } - if compressed.len() < 2 { - return Err(CompressionError::InvalidCompressedData); - } - - let is_delta = compressed.get(1).unwrap() != 0; - - if is_delta { - decompress_delta_encoded(env, compressed, max_decompressed_size) - } else { - decompress_rle_encoded(env, compressed, max_decompressed_size) - } + Ok(metadata.clone()) } -/// Check if data is already compressed with our format. +/// Return whether metadata uses the current or legacy compressed format. pub fn is_compressed(metadata: &Bytes) -> bool { - !metadata.is_empty() && metadata.get(0).unwrap() == COMPRESSION_MAGIC + matches!(parse_messagepack_header(metadata), Ok(Some(_))) || is_legacy_compressed(metadata) } -// ============================================================================ -// Private: Delta + RLE Compression -// ============================================================================ +fn encode_blocks(env: &Env, metadata: &Bytes, mode: u8) -> Bytes { + let mut payload = Bytes::new(env); + payload.push_back(FORMAT_VERSION); + payload.push_back(mode); + + let mut index = 0u32; + while index < metadata.len() { + let repeated = repeat_length(metadata, index, mode); + if repeated >= 3 { + payload.push_back(0x80 | ((repeated - 3) as u8)); + payload.push_back(transformed_byte(metadata, index, mode)); + index += repeated; + continue; + } -/// Delta encoding: store differences from the previous byte. -/// Works well for structured data like JSON. -fn compress_with_delta( - env: &Env, - metadata: &Bytes, - max_size: u32, -) -> Result { - let mut out = Bytes::new(env); - out.push_back(COMPRESSION_MAGIC); - out.push_back(1); // is_delta_encoded = true + let literal_start = index; + let mut literal_length = 0u32; + while index < metadata.len() && literal_length < MAX_LITERAL_LENGTH { + if literal_length > 0 && repeat_length(metadata, index, mode) >= 3 { + break; + } + index += 1; + literal_length += 1; + } - let len = metadata.len(); - let mut prev: i16 = 0; // Use i16 to avoid overflow on subtraction + payload.push_back((literal_length - 1) as u8); + for literal_index in literal_start..(literal_start + literal_length) { + payload.push_back(transformed_byte(metadata, literal_index, mode)); + } + } - let mut i: u32 = 0; - while i < len { - let current = metadata.get(i).unwrap() as i16; - let delta = (current - prev) as i8; - out.push_back(delta as u8); + payload +} - if out.len() >= max_size { - return Err(CompressionError::OutputTooLarge); - } +fn repeat_length(metadata: &Bytes, start: u32, mode: u8) -> u32 { + let expected = transformed_byte(metadata, start, mode); + let mut length = 1u32; + while length < MAX_REPEAT_LENGTH + && start + length < metadata.len() + && transformed_byte(metadata, start + length, mode) == expected + { + length += 1; + } + length +} - prev = current; - i += 1; +fn transformed_byte(metadata: &Bytes, index: u32, mode: u8) -> u8 { + let current = metadata.get(index).unwrap(); + if mode == MODE_DELTA { + let previous = if index == 0 { + 0 + } else { + metadata.get(index - 1).unwrap() + }; + current.wrapping_sub(previous) + } else { + current } +} - Ok(out) +fn frame_messagepack_extension(env: &Env, payload: &Bytes) -> Bytes { + let mut framed = Bytes::new(env); + if payload.len() <= u8::MAX as u32 { + framed.push_back(MESSAGEPACK_EXT8); + framed.push_back(payload.len() as u8); + } else { + framed.push_back(MESSAGEPACK_EXT16); + framed.push_back((payload.len() >> 8) as u8); + framed.push_back(payload.len() as u8); + } + framed.push_back(ETHOS_METADATA_TYPE); + framed.append(payload); + framed } -/// RLE encoding for runs of identical bytes. -fn compress_with_rle( - env: &Env, +fn parse_messagepack_header( metadata: &Bytes, - max_size: u32, -) -> Result { - let mut out = Bytes::new(env); - out.push_back(COMPRESSION_MAGIC); - out.push_back(0); // is_delta_encoded = false - - let len = metadata.len(); - let mut i: u32 = 0; - - while i < len { - let current = metadata.get(i).unwrap(); - let mut run: u32 = 1; +) -> Result, CompressionError> { + if metadata.is_empty() { + return Ok(None); + } - // Count consecutive identical bytes (capped at 255) - while run < 255 && (i + run) < len && metadata.get(i + run).unwrap() == current { - run += 1; + let (payload_start, payload_length) = match metadata.get(0).unwrap() { + MESSAGEPACK_EXT8 + if metadata.len() >= 3 && metadata.get(2).unwrap() == ETHOS_METADATA_TYPE => + { + (3u32, metadata.get(1).unwrap() as u32) } - - // Encode as (count, byte) pair - out.push_back(run as u8); - out.push_back(current); - - if out.len() >= max_size { - return Err(CompressionError::OutputTooLarge); + MESSAGEPACK_EXT16 + if metadata.len() >= 4 && metadata.get(3).unwrap() == ETHOS_METADATA_TYPE => + { + let length = + ((metadata.get(1).unwrap() as u32) << 8) | metadata.get(2).unwrap() as u32; + (4u32, length) } + _ => return Ok(None), + }; - i += run; + let payload_end = payload_start + .checked_add(payload_length) + .ok_or(CompressionError::InvalidCompressedData)?; + if payload_length < 2 || payload_end != metadata.len() { + return Err(CompressionError::InvalidCompressedData); } - Ok(out) -} + let version = metadata.get(payload_start).unwrap(); + let mode = metadata.get(payload_start + 1).unwrap(); + if version != FORMAT_VERSION || (mode != MODE_DIRECT && mode != MODE_DELTA) { + return Err(CompressionError::InvalidCompressedData); + } -// ============================================================================ -// Private: Delta + RLE Decompression -// ============================================================================ + Ok(Some((payload_start + 2, payload_end, mode))) +} -fn decompress_delta_encoded( +fn decode_blocks( env: &Env, - compressed: &Bytes, - max_size: u32, + encoded: &Bytes, + mut index: u32, + end: u32, + mode: u8, ) -> Result { - let mut out = Bytes::new(env); - let mut current: i16 = 0; - let compressed_len = compressed.len(); - - let mut i: u32 = 2; // Skip magic + flag - while i < compressed_len { - let delta = compressed.get(i).unwrap() as i8 as i16; - current = current.wrapping_add(delta); + let mut output = Bytes::new(env); + let mut previous = 0u8; - let byte = (current & 0xFF) as u8; - out.push_back(byte); + while index < end { + let token = encoded.get(index).unwrap(); + index += 1; - if out.len() >= max_size { - return Err(CompressionError::OutputTooLarge); + if token & 0x80 != 0 { + if index >= end { + return Err(CompressionError::InvalidCompressedData); + } + let count = (token as u32 & 0x7F) + 3; + let value = encoded.get(index).unwrap(); + index += 1; + for _ in 0..count { + push_decoded(&mut output, value, mode, &mut previous)?; + } + } else { + let count = (token as u32 & 0x7F) + 1; + if index + count > end { + return Err(CompressionError::InvalidCompressedData); + } + for _ in 0..count { + let value = encoded.get(index).unwrap(); + index += 1; + push_decoded(&mut output, value, mode, &mut previous)?; + } } + } - i += 1; + Ok(output) +} + +fn push_decoded( + output: &mut Bytes, + value: u8, + mode: u8, + previous: &mut u8, +) -> Result<(), CompressionError> { + if output.len() >= MAX_METADATA_SIZE { + return Err(CompressionError::OutputTooLarge); } - Ok(out) + let decoded = if mode == MODE_DELTA { + previous.wrapping_add(value) + } else { + value + }; + output.push_back(decoded); + *previous = decoded; + Ok(()) } -fn decompress_rle_encoded( - env: &Env, - compressed: &Bytes, - max_size: u32, -) -> Result { - let mut out = Bytes::new(env); - let compressed_len = compressed.len(); +fn is_legacy_compressed(metadata: &Bytes) -> bool { + metadata.len() >= 2 + && metadata.get(0).unwrap() == LEGACY_MAGIC + && matches!(metadata.get(1).unwrap(), MODE_DIRECT | MODE_DELTA) +} - let mut i: u32 = 2; // Skip magic + flag - while i < compressed_len { - if i + 1 >= compressed_len { - return Err(CompressionError::InvalidCompressedData); +fn decompress_legacy(env: &Env, metadata: &Bytes) -> Result { + let mode = metadata.get(1).unwrap(); + if mode == MODE_DELTA { + let mut output = Bytes::new(env); + let mut previous = 0u8; + for index in 2..metadata.len() { + push_decoded( + &mut output, + metadata.get(index).unwrap(), + MODE_DELTA, + &mut previous, + )?; } + return Ok(output); + } - let count = compressed.get(i).unwrap(); - let byte = compressed.get(i + 1).unwrap(); - + let mut output = Bytes::new(env); + let mut index = 2u32; + let mut previous = 0u8; + while index < metadata.len() { + if index + 1 >= metadata.len() { + return Err(CompressionError::InvalidCompressedData); + } + let count = metadata.get(index).unwrap(); + let value = metadata.get(index + 1).unwrap(); + if count == 0 { + return Err(CompressionError::InvalidCompressedData); + } for _ in 0..count { - out.push_back(byte); - - if out.len() >= max_size { - return Err(CompressionError::OutputTooLarge); - } + push_decoded(&mut output, value, MODE_DIRECT, &mut previous)?; } - - i += 2; + index += 2; } - Ok(out) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_compression_magic() { - assert_eq!(COMPRESSION_MAGIC, 0xC1); - } + Ok(output) } diff --git a/contracts/sbt/src/compression_tests.rs b/contracts/sbt/src/compression_tests.rs new file mode 100644 index 0000000..9971c12 --- /dev/null +++ b/contracts/sbt/src/compression_tests.rs @@ -0,0 +1,149 @@ +#![cfg(test)] + +use super::*; +use soroban_sdk::testutils::Address as _; + +fn setup() -> (Env, Address, Address) { + let env = Env::default(); + env.mock_all_auths(); + + let admin = Address::generate(&env); + let owner = Address::generate(&env); + let contract_id = env.register_contract(None, SbtContract); + SbtContractClient::new(&env, &contract_id).initialize(&admin); + + (env, contract_id, owner) +} + +#[test] +fn messagepack_compression_round_trips_repeated_metadata() { + let (env, contract_id, _) = setup(); + let client = SbtContractClient::new(&env, &contract_id); + let metadata = Bytes::from_slice(&env, &[b'A'; 64]); + + let compressed = client.compress_metadata(&metadata); + + assert!(compressed.len() < metadata.len()); + assert_eq!(compressed.get(0).unwrap(), 0xC7); + assert_eq!(client.decompress_metadata(&compressed), metadata); +} + +#[test] +fn messagepack_ext16_round_trips_larger_compressed_metadata() { + let (env, contract_id, _) = setup(); + let client = SbtContractClient::new(&env, &contract_id); + let mut metadata = Bytes::new(&env); + for value in 0..134u32 { + let byte = (value % 251) as u8; + metadata.push_back(byte); + metadata.push_back(byte); + metadata.push_back(byte); + } + + let compressed = client.compress_metadata(&metadata); + + assert!(compressed.len() < metadata.len()); + assert_eq!(compressed.get(0).unwrap(), 0xC8); + assert_eq!(client.decompress_metadata(&compressed), metadata); +} + +#[test] +fn compression_keeps_incompressible_metadata_unframed() { + let (env, contract_id, _) = setup(); + let client = SbtContractClient::new(&env, &contract_id); + let metadata = Bytes::from_slice(&env, &[0, 17, 53, 102, 166, 245, 77, 190]); + + let compressed = client.compress_metadata(&metadata); + + assert_eq!(compressed, metadata); + assert!(!compression::is_compressed(&compressed)); +} + +#[test] +fn decompression_is_backwards_compatible_with_raw_metadata() { + let (env, contract_id, _) = setup(); + let client = SbtContractClient::new(&env, &contract_id); + let metadata = Bytes::from_slice(&env, b"existing uncompressed metadata"); + + assert_eq!(client.decompress_metadata(&metadata), metadata); +} + +#[test] +fn decompression_supports_legacy_delta_and_rle_metadata() { + let (env, contract_id, _) = setup(); + let client = SbtContractClient::new(&env, &contract_id); + let legacy_delta = Bytes::from_slice(&env, &[0xC1, 1, b'A', 0, 0]); + let legacy_rle = Bytes::from_slice(&env, &[0xC1, 0, 3, b'A']); + let expected = Bytes::from_slice(&env, b"AAA"); + + assert_eq!(client.decompress_metadata(&legacy_delta), expected); + assert_eq!(client.decompress_metadata(&legacy_rle), expected); +} + +#[test] +fn malformed_ethos_messagepack_extension_is_rejected() { + let (env, contract_id, _) = setup(); + let client = SbtContractClient::new(&env, &contract_id); + let malformed = Bytes::from_slice(&env, &[0xC7, 3, 0x45, 1, 0, 0]); + + assert!(client.try_decompress_metadata(&malformed).is_err()); +} + +#[test] +fn empty_metadata_round_trips_without_a_header() { + let (env, contract_id, _) = setup(); + let client = SbtContractClient::new(&env, &contract_id); + let metadata = Bytes::new(&env); + + assert_eq!(client.compress_metadata(&metadata), metadata); + assert_eq!(client.decompress_metadata(&metadata), metadata); +} + +#[test] +fn in_place_compression_preserves_the_string_metadata_api() { + let (env, contract_id, owner) = setup(); + let client = SbtContractClient::new(&env, &contract_id); + let metadata = String::from_str( + &env, + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ); + let credential_id = client.mint(&owner, &metadata); + + let bytes_saved = client.compress_sbt_metadata(&credential_id); + + assert_eq!(bytes_saved, 57); + assert!(client.is_sbt_metadata_compressed(&credential_id)); + assert_eq!(client.get_metadata(&credential_id), metadata); + assert_eq!( + client.decompress_sbt_metadata(&credential_id), + Bytes::from_slice(&env, &[b'A'; 64]) + ); + assert_eq!(client.compress_sbt_metadata(&credential_id), 0); +} + +#[test] +fn in_place_compression_leaves_larger_encodings_unmodified() { + let (env, contract_id, owner) = setup(); + let client = SbtContractClient::new(&env, &contract_id); + let metadata = String::from_str(&env, "short"); + let credential_id = client.mint(&owner, &metadata); + + assert_eq!(client.compress_sbt_metadata(&credential_id), 0); + assert!(!client.is_sbt_metadata_compressed(&credential_id)); + assert_eq!(client.get_metadata(&credential_id), metadata); +} + +#[test] +fn benchmark_repeated_metadata_storage_savings() { + let (env, contract_id, _) = setup(); + let client = SbtContractClient::new(&env, &contract_id); + let metadata = Bytes::from_slice(&env, &[b'A'; 64]); + let compressed = client.compress_metadata(&metadata); + let bytes_saved = metadata.len() - compressed.len(); + let savings_bps = bytes_saved * 10_000 / metadata.len(); + + assert_eq!(metadata.len(), 64); + assert_eq!(compressed.len(), 7); + assert_eq!(bytes_saved, 57); + assert_eq!(savings_bps, 8_906); +} diff --git a/contracts/sbt/src/lib.rs b/contracts/sbt/src/lib.rs index 1fc001c..449adf9 100644 --- a/contracts/sbt/src/lib.rs +++ b/contracts/sbt/src/lib.rs @@ -4,11 +4,14 @@ mod compression; #[cfg(test)] mod atomic_release_tests; +use crate::compression::{ + compress_metadata as compress_metadata_bytes, + decompress_metadata as decompress_metadata_bytes, is_compressed, MAX_METADATA_SIZE, +}; use soroban_sdk::{ contract, contractclient, contracterror, contractimpl, contracttype, panic_with_error, symbol_short, Address, Bytes, Env, Map, String, Vec, }; -use crate::compression::{compress_metadata, decompress_metadata, is_compressed}; const MINT_TOPIC: soroban_sdk::Symbol = symbol_short!("sbt_mint"); const COMPOSE_TOPIC: soroban_sdk::Symbol = symbol_short!("sbt_cmpse"); @@ -270,29 +273,57 @@ impl SbtContract { } pub fn get_metadata(env: Env, sbt_id: u64) -> String { - env.storage() + if !Self::is_sbt_metadata_compressed(env.clone(), sbt_id) { + return env + .storage() + .instance() + .get(&DataKey::Metadata(sbt_id)) + .unwrap_or_else(|| panic_with_error!(&env, SbtError::TokenNotFound)); + } + + let compressed: Bytes = env + .storage() .instance() .get(&DataKey::Metadata(sbt_id)) - .unwrap_or_else(|| panic_with_error!(&env, SbtError::TokenNotFound)) + .unwrap_or_else(|| panic_with_error!(&env, SbtError::TokenNotFound)); + let metadata = decompress_metadata_bytes(&env, &compressed) + .unwrap_or_else(|_| panic_with_error!(&env, SbtError::MetadataCompressionFailed)); + Self::bytes_to_string(&env, &metadata) + } + + /// Compress arbitrary credential metadata using the Ethos MessagePack format. + /// + /// Metadata that would not become smaller is returned unchanged. + pub fn compress_metadata(env: Env, metadata: Bytes) -> Bytes { + compress_metadata_bytes(&env, &metadata) + } + + /// Decompress current, legacy, or ordinary uncompressed credential metadata. + pub fn decompress_metadata(env: Env, metadata: Bytes) -> Bytes { + decompress_metadata_bytes(&env, &metadata) + .unwrap_or_else(|_| panic_with_error!(&env, SbtError::MetadataCompressionFailed)) } /// Compress an SBT's metadata in-place. Owner only. pub fn compress_sbt_metadata(env: Env, sbt_id: u64) -> u64 { Self::require_owner(&env, sbt_id); - - let metadata: Vec = env + + if Self::is_sbt_metadata_compressed(env.clone(), sbt_id) { + return 0; + } + + let metadata: String = env .storage() .instance() .get(&DataKey::Metadata(sbt_id)) .unwrap_or_else(|| panic_with_error!(&env, SbtError::TokenNotFound)); + let metadata = Self::string_to_bytes(&env, &metadata); + let compressed = compress_metadata_bytes(&env, &metadata); - if is_compressed(&metadata) { + if !is_compressed(&compressed) { return 0; } - let compressed = compress_metadata(&env, &metadata, 4096) - .unwrap_or_else(|_| panic_with_error!(&env, SbtError::MetadataCompressionFailed)); - let original_size = metadata.len() as u64; let compressed_size = compressed.len() as u64; @@ -310,18 +341,23 @@ impl SbtContract { } /// Decompress an SBT's metadata if it was compressed, returning raw bytes. - pub fn decompress_sbt_metadata(env: Env, sbt_id: u64) -> Vec { - let metadata: Vec = env + pub fn decompress_sbt_metadata(env: Env, sbt_id: u64) -> Bytes { + if !Self::is_sbt_metadata_compressed(env.clone(), sbt_id) { + let metadata: String = env + .storage() + .instance() + .get(&DataKey::Metadata(sbt_id)) + .unwrap_or_else(|| panic_with_error!(&env, SbtError::TokenNotFound)); + return Self::string_to_bytes(&env, &metadata); + } + + let metadata: Bytes = env .storage() .instance() .get(&DataKey::Metadata(sbt_id)) .unwrap_or_else(|| panic_with_error!(&env, SbtError::TokenNotFound)); - if !is_compressed(&metadata) { - return metadata; - } - - decompress_metadata(&env, &metadata, 4096) + decompress_metadata_bytes(&env, &metadata) .unwrap_or_else(|_| panic_with_error!(&env, SbtError::MetadataCompressionFailed)) } @@ -744,6 +780,28 @@ impl SbtContract { admin.require_auth(); } + fn string_to_bytes(env: &Env, metadata: &String) -> Bytes { + if metadata.len() > MAX_METADATA_SIZE { + panic_with_error!(env, SbtError::MetadataCompressionFailed); + } + + let length = metadata.len() as usize; + let mut buffer = [0u8; MAX_METADATA_SIZE as usize]; + metadata.copy_into_slice(&mut buffer[..length]); + Bytes::from_slice(env, &buffer[..length]) + } + + fn bytes_to_string(env: &Env, metadata: &Bytes) -> String { + if metadata.len() > MAX_METADATA_SIZE { + panic_with_error!(env, SbtError::MetadataCompressionFailed); + } + + let length = metadata.len() as usize; + let mut buffer = [0u8; MAX_METADATA_SIZE as usize]; + metadata.copy_into_slice(&mut buffer[..length]); + String::from_bytes(env, &buffer[..length]) + } + fn load_owner(env: &Env, sbt_id: u64) -> Address { env.storage() .instance() diff --git a/docs/sbt-advanced-features.md b/docs/sbt-advanced-features.md index 771c2a2..b4e3345 100644 --- a/docs/sbt-advanced-features.md +++ b/docs/sbt-advanced-features.md @@ -2,7 +2,7 @@ This document covers the implementation of three advanced features for Soroban SBT (Soul-Bound Token) contracts: -1. **Metadata Compression** (Issue #47) +1. **Metadata Compression** (Issue #26, superseding the legacy #47 format) 2. **Fractional Ownership** (Issue #45) 3. **SBT Escrow for Conditional Transfer** (Issue #46) @@ -12,29 +12,53 @@ This document covers the implementation of three advanced features for Soroban S ### Overview -SBT metadata can grow large when storing structured data (JSON, images URIs, attributes, etc.). Metadata compression reduces on-chain storage costs using delta encoding + run-length encoding (RLE). +SBT metadata can grow large when storing structured data (JSON, image URIs, +attributes, etc.). Metadata compression reduces on-chain storage costs with a +MessagePack extension containing a compact PackBits-style block stream. ### How It Works #### Compression Strategy -The compression module uses two techniques: +The compressor creates direct-byte and delta-byte candidates. Each candidate +uses literal and repeated-byte blocks, and the smaller candidate is wrapped in +a MessagePack extension. If the complete framed value is not smaller than the +original metadata, the original bytes are returned unchanged. -1. **Delta Encoding**: Store differences between consecutive bytes rather than absolute values. This is particularly effective for JSON metadata where many fields have similar structures. +#### Compression Format -2. **Run-Length Encoding (RLE)**: After delta encoding, compress runs of identical bytes. Common in padding, repeated quotes, and structural elements. +```text +MessagePack extension: + [0xC7][payload length: u8][type: 0x45][payload...] + or + [0xC8][payload length: u16 big-endian][type: 0x45][payload...] -#### Compression Format +Payload: + [version: 0x01][mode: 0x00 direct | 0x01 delta][blocks...] -``` -Compressed Data: - [MAGIC: 0xC1] — First byte (marks as compressed metadata) - [is_delta: u8] — 0 or 1 (encoding method) - [data...] — Encoded payload +Block: + Literal: [0LLLLLLL][1-128 literal bytes] + Repeat: [1RRRRRRR][one repeated byte] + +Literal length = L + 1 +Repeat length = R + 3 ``` +The extension type `0x45` identifies Ethos credential metadata. Ext8 is used +for payloads up to 255 bytes; larger payloads use Ext16. + ### API +#### Compress and Decompress Bytes + +```rust +pub fn compress_metadata(env: Env, metadata: Bytes) -> Bytes +pub fn decompress_metadata(env: Env, metadata: Bytes) -> Bytes +``` + +`decompress_metadata` accepts the current MessagePack extension, the legacy +`0xC1` delta/RLE representation, and ordinary uncompressed bytes. + #### Create (at mint time) New SBTs are created with uncompressed metadata: @@ -54,6 +78,7 @@ pub fn compress_sbt_metadata(env: Env, sbt_id: u64) -> u64 { // Returns: size reduction (original_size - compressed_size) // Only owner can compress // Idempotent: already compressed SBTs return 0 + // Metadata remains unmodified when compression would increase its size } ``` @@ -61,11 +86,10 @@ pub fn compress_sbt_metadata(env: Env, sbt_id: u64) -> u64 { ```rust pub fn get_metadata(env: Env, sbt_id: u64) -> String { - // Returns: raw (uncompressed) metadata - // Automatically decompresses if needed + // Preserves the existing String API and automatically decompresses } -pub fn decompress_sbt_metadata(env: Env, sbt_id: u64) -> Vec { +pub fn decompress_sbt_metadata(env: Env, sbt_id: u64) -> Bytes { // Returns: raw bytes (decompressed if compressed, as-is otherwise) } ``` @@ -84,16 +108,22 @@ pub fn is_sbt_metadata_compressed(env: Env, sbt_id: u64) -> bool { - New SBTs default to uncompressed format - Compression is opt-in via `compress_sbt_metadata()` - Decompression is automatic on read operations +- Metadata written with the legacy `0xC1` representation remains readable +- Unknown or ordinary bytes pass through decompression unchanged ### Storage Savings -Typical JSON metadata achieves 40-60% compression: +The focused storage benchmark uses 64 repeated metadata bytes: -``` -Example: - Original: {"name":"Alice","age":28,"country":"US","verified":true} = 69 bytes - Compressed: Typically 28-35 bytes (60%+ savings) -``` +| Representation | Bytes | +| --- | ---: | +| Original | 64 | +| MessagePack extension | 7 | +| Saved | 57 | +| Savings | 89.06% | + +Savings depend on the input. Short or non-repeating metadata remains +uncompressed when the MessagePack envelope would be larger. ---