From 4315428e004068668b97e70e52b5b418e2a5f921 Mon Sep 17 00:00:00 2001 From: Handy-caT <37216852+Handy-caT@users.noreply.github.com> Date: Fri, 4 Apr 2025 16:33:02 +0300 Subject: [PATCH 1/8] start page for unsized --- src/lib.rs | 6 +- src/page/index/mod.rs | 32 ++++- src/page/index/page.rs | 100 ++++++------- src/page/index/page_for_unsized.rs | 218 +++++++++++++++++++++++++++++ src/page/mod.rs | 5 +- 5 files changed, 299 insertions(+), 62 deletions(-) create mode 100644 src/page/index/page_for_unsized.rs diff --git a/src/lib.rs b/src/lib.rs index e0d2713..978ab14 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,9 +12,9 @@ pub use data_bucket_codegen::SizeMeasure; pub use page::{ get_index_page_size_from_data_length, map_data_pages_to_general, parse_data_page, parse_general_header_by_index, parse_page, persist_page, seek_by_link, seek_to_page_start, - update_at, DataPage, GeneralHeader, GeneralPage, IndexPage, IndexValue, Interval, PageType, - SpaceInfoPage, TableOfContentsPage, DATA_VERSION, GENERAL_HEADER_SIZE, INNER_PAGE_SIZE, - PAGE_SIZE, + update_at, DataPage, GeneralHeader, GeneralPage, IndexPage, IndexPageUtility, IndexValue, + Interval, PageType, SpaceInfoPage, TableOfContentsPage, UnsizedIndexPage, DATA_VERSION, + GENERAL_HEADER_SIZE, INNER_PAGE_SIZE, PAGE_SIZE, }; pub use persistence::{PersistableIndex, PersistableTable}; pub use space::Id as SpaceId; diff --git a/src/page/index/mod.rs b/src/page/index/mod.rs index 7500296..f08e622 100644 --- a/src/page/index/mod.rs +++ b/src/page/index/mod.rs @@ -1,14 +1,44 @@ -use crate::{Link, SizeMeasurable}; +use std::io::SeekFrom; + use indexset::core::multipair::MultiPair; use indexset::core::pair::Pair; use rkyv::{Archive, Deserialize, Serialize}; +use tokio::fs::File; +use tokio::io::{AsyncSeekExt, AsyncWriteExt}; + +use crate::{seek_to_page_start, Link, Persistable, SizeMeasurable, GENERAL_HEADER_SIZE}; mod page; +mod page_for_unsized; mod table_of_contents_page; +use crate::page::PageId; + pub use page::{get_index_page_size_from_data_length, IndexPage}; +pub use page_for_unsized::UnsizedIndexPage; pub use table_of_contents_page::TableOfContentsPage; +pub trait IndexPageUtility { + type Utility: Persistable; + + async fn parse_index_page_utility( + file: &mut File, + page_id: PageId, + ) -> eyre::Result; + + async fn persist_index_page_utility( + file: &mut File, + page_id: PageId, + utility: Self::Utility, + ) -> eyre::Result<()> { + seek_to_page_start(file, page_id.0).await?; + file.seek(SeekFrom::Current(GENERAL_HEADER_SIZE as i64)) + .await?; + file.write_all(utility.as_bytes().as_ref()).await?; + Ok(()) + } +} + /// Represents `key/value` pair of B-Tree index, where value is always /// [`data::Link`], as it is represented in primary and secondary indexes. #[derive( diff --git a/src/page/index/page.rs b/src/page/index/page.rs index 0cacc1d..12c8129 100644 --- a/src/page/index/page.rs +++ b/src/page/index/page.rs @@ -17,6 +17,7 @@ use rkyv::{Archive, Deserialize, Serialize}; use tokio::fs::File; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; +use crate::page::index::IndexPageUtility; use crate::page::{IndexValue, PageId}; use crate::{ align, align8, seek_to_page_start, Link, Persistable, SizeMeasurable, GENERAL_HEADER_SIZE, @@ -63,7 +64,7 @@ pub struct IndexPage { Archive, Clone, Deserialize, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Persistable, )] #[persistable(by_parts)] -pub struct IndexPageUtility { +pub struct SizedIndexPageUtility { pub size: u16, pub node_id: T, pub current_index: u16, @@ -71,6 +72,47 @@ pub struct IndexPageUtility { pub slots: Vec, } +impl IndexPageUtility for IndexPage +where + T: Archive + + for<'a> Serialize< + Strategy, Share>, rkyv::rancor::Error>, + >, + ::Archived: Deserialize>, +{ + type Utility = SizedIndexPageUtility; + + async fn parse_index_page_utility( + file: &mut File, + page_id: PageId, + ) -> eyre::Result { + seek_to_page_start(file, page_id.0).await?; + let offset = GENERAL_HEADER_SIZE as i64; + file.seek(SeekFrom::Current(offset)).await?; + + let mut size_bytes = vec![0u8; SizedIndexPageUtility::::size_size()]; + file.read_exact(size_bytes.as_mut_slice()).await?; + let archived = unsafe { + rkyv::access_unchecked::<::Archived>( + &size_bytes[0..SizedIndexPageUtility::::size_size()], + ) + }; + let size = + rkyv::deserialize::(archived).expect("data should be valid"); + + let index_utility_len = SizedIndexPageUtility::::persisted_size(size as usize); + file.seek(SeekFrom::Current( + -(SizedIndexPageUtility::::size_size() as i64), + )) + .await?; + let mut index_utility_bytes = vec![0u8; index_utility_len]; + file.read_exact(index_utility_bytes.as_mut_slice()).await?; + let utility = SizedIndexPageUtility::::from_bytes(&index_utility_bytes); + + Ok(utility) + } +} + impl IndexPage { pub fn new(node_id: T, size: usize) -> Self where @@ -116,62 +158,6 @@ impl IndexPage { new_page } - pub async fn parse_index_page_utility( - file: &mut File, - page_id: PageId, - ) -> eyre::Result> - where - T: Archive - + for<'a> Serialize< - Strategy, Share>, rkyv::rancor::Error>, - >, - ::Archived: Deserialize>, - { - seek_to_page_start(file, page_id.0).await?; - let offset = GENERAL_HEADER_SIZE as i64; - file.seek(SeekFrom::Current(offset)).await?; - - let mut size_bytes = vec![0u8; IndexPageUtility::::size_size()]; - file.read_exact(size_bytes.as_mut_slice()).await?; - let archived = unsafe { - rkyv::access_unchecked::<::Archived>( - &size_bytes[0..IndexPageUtility::::size_size()], - ) - }; - let size = - rkyv::deserialize::(archived).expect("data should be valid"); - - let index_utility_len = IndexPageUtility::::persisted_size(size as usize); - file.seek(SeekFrom::Current( - -(IndexPageUtility::::size_size() as i64), - )) - .await?; - let mut index_utility_bytes = vec![0u8; index_utility_len]; - file.read_exact(index_utility_bytes.as_mut_slice()).await?; - let utility = IndexPageUtility::::from_bytes(&index_utility_bytes); - - Ok(utility) - } - - pub async fn persist_index_page_utility( - file: &mut File, - page_id: PageId, - utility: IndexPageUtility, - ) -> eyre::Result<()> - where - T: Archive - + for<'a> Serialize< - Strategy, Share>, rkyv::rancor::Error>, - >, - ::Archived: Deserialize>, - { - seek_to_page_start(file, page_id.0).await?; - file.seek(SeekFrom::Current(GENERAL_HEADER_SIZE as i64)) - .await?; - file.write_all(utility.as_bytes().as_ref()).await?; - Ok(()) - } - async fn read_value(file: &mut File) -> eyre::Result> where T: Archive, diff --git a/src/page/index/page_for_unsized.rs b/src/page/index/page_for_unsized.rs new file mode 100644 index 0000000..3d67f2f --- /dev/null +++ b/src/page/index/page_for_unsized.rs @@ -0,0 +1,218 @@ +use data_bucket_codegen::Persistable; +use rkyv::de::Pool; +use rkyv::rancor::Strategy; +use rkyv::ser::allocator::ArenaHandle; +use rkyv::ser::sharing::Share; +use rkyv::ser::Serializer; +use rkyv::util::AlignedVec; +use rkyv::{Archive, Deserialize, Serialize}; +use std::io::SeekFrom; +use tokio::fs::File; +use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; + +use crate::align8; +use crate::page::index::IndexPageUtility; +use crate::page::PageId; +use crate::Persistable; +use crate::{seek_to_page_start, IndexValue, SizeMeasurable, GENERAL_HEADER_SIZE}; + +#[derive(Archive, Clone, Deserialize, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +pub struct UnsizedIndexPage { + pub size: u16, + pub node_id: T, + pub last_value_offset: u32, + pub slots: Vec<(u32, u16)>, + pub index_values: Vec>, +} + +#[derive( + Archive, Clone, Deserialize, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Persistable, +)] +#[persistable(by_parts)] +pub struct UnsizedIndexPageUtility { + pub size: u16, + pub node_id: T, + pub last_value_offset: u32, + pub slots: Vec<(u32, u16)>, +} + +impl IndexPageUtility + for UnsizedIndexPage +where + T: Archive + + for<'a> Serialize< + Strategy, Share>, rkyv::rancor::Error>, + >, + ::Archived: Deserialize>, +{ + type Utility = UnsizedIndexPageUtility; + + async fn parse_index_page_utility( + file: &mut File, + page_id: PageId, + ) -> eyre::Result { + seek_to_page_start(file, page_id.0).await?; + let offset = GENERAL_HEADER_SIZE as i64; + file.seek(SeekFrom::Current(offset)).await?; + + let mut size_bytes = vec![0u8; UnsizedIndexPageUtility::::size_size()]; + file.read_exact(size_bytes.as_mut_slice()).await?; + let archived = unsafe { + rkyv::access_unchecked::<::Archived>( + &size_bytes[0..UnsizedIndexPageUtility::::size_size()], + ) + }; + let size = + rkyv::deserialize::(archived).expect("data should be valid"); + + let index_utility_len = UnsizedIndexPageUtility::::persisted_size(size as usize); + file.seek(SeekFrom::Current( + -(UnsizedIndexPageUtility::::size_size() as i64), + )) + .await?; + let mut index_utility_bytes = vec![0u8; index_utility_len]; + file.read_exact(index_utility_bytes.as_mut_slice()).await?; + let utility = UnsizedIndexPageUtility::::from_bytes(&index_utility_bytes); + + Ok(utility) + } +} + +impl UnsizedIndexPage +where + T: Archive + + Default + + SizeMeasurable + + for<'a> Serialize< + Strategy, Share>, rkyv::rancor::Error>, + >, + ::Archived: Deserialize>, +{ + pub fn new(node_id: T, value: IndexValue) -> eyre::Result { + let bytes = rkyv::to_bytes::(&value)?; + let len = bytes.len() as u32; + Ok(Self { + size: 1, + node_id, + last_value_offset: len, + slots: vec![(len, len as u16)], + index_values: vec![value], + }) + } + + pub async fn persist_value( + file: &mut File, + page_id: PageId, + current_offset: u32, + value: IndexValue, + ) -> eyre::Result + where + T: Archive + + Eq + + for<'a> Serialize< + Strategy, Share>, rkyv::rancor::Error>, + >, + ::Archived: Deserialize>, + { + // We seek to page's end and will write values from tail. + seek_to_page_start(file, page_id.0 + 1).await?; + + let bytes = rkyv::to_bytes::(&value)?; + let offset = current_offset + bytes.len() as u32; + file.seek(SeekFrom::Current(offset as i64)).await?; + file.write_all(bytes.as_slice()).await?; + + Ok(offset) + } +} + +impl Persistable for UnsizedIndexPage +where + T: Archive + + Clone + + Default + + SizeMeasurable + + for<'a> Serialize< + Strategy, Share>, rkyv::rancor::Error>, + >, + ::Archived: Deserialize>, +{ + fn as_bytes(&self) -> impl AsRef<[u8]> + Send { + let data_length = DATA_LENGTH as usize; + let utility = UnsizedIndexPageUtility { + size: self.size, + node_id: self.node_id.clone(), + last_value_offset: self.last_value_offset, + slots: self.slots.clone(), + }; + let utility_bytes = utility.as_bytes(); + let utility_bytes = utility_bytes.as_ref().to_vec(); + let utility_len = utility_bytes.len(); + let mut bytes = vec![0u8; data_length]; + bytes.splice(0..utility_len, utility_bytes.iter().map(|v| *v)); + + for ((offset, len), value) in self.slots.iter().zip(self.index_values.iter()) { + let offset = data_length - *offset as usize; + let len = *len as usize; + let value_bytes = rkyv::to_bytes::(value).unwrap(); + bytes.splice(offset..(offset + len), value_bytes.iter().map(|v| *v)); + } + + bytes + } + + fn from_bytes(bytes: &[u8]) -> Self { + let size_bytes = &bytes[0..UnsizedIndexPageUtility::::size_size()]; + let archived = unsafe { + rkyv::access_unchecked::<::Archived>( + &size_bytes[0..UnsizedIndexPageUtility::::size_size()], + ) + }; + let size = + rkyv::deserialize::(archived).expect("data should be valid"); + let utility_len = UnsizedIndexPageUtility::::persisted_size(size as usize); + let utility = UnsizedIndexPageUtility::::from_bytes(&bytes[0..utility_len]); + let mut index_values = Vec::with_capacity(utility.slots.len()); + for (offset, len) in &utility.slots { + let offset = bytes.len() - *offset as usize; + let len = *len as usize; + let value_bytes = &bytes[offset..(offset + len)]; + let archived = unsafe { + rkyv::access_unchecked::< as Archive>::Archived>(value_bytes) + }; + let val = rkyv::deserialize::<_, rkyv::rancor::Error>(archived) + .expect("data should be valid"); + index_values.push(val) + } + + Self { + size, + node_id: utility.node_id, + last_value_offset: utility.last_value_offset, + slots: utility.slots, + index_values, + } + } +} + +#[cfg(test)] +mod test { + use crate::{IndexValue, Link, Persistable, UnsizedIndexPage}; + + #[test] + fn to_bytes_and_back() { + let value = IndexValue { + key: "SomeID".to_string(), + link: Link { + page_id: 0.into(), + offset: 0, + length: 40, + }, + }; + let page = UnsizedIndexPage::<_, 1024>::new("SomeID".to_string(), value).unwrap(); + let bytes = page.as_bytes(); + assert_eq!(bytes.as_ref().len(), 1024); + let page_back = UnsizedIndexPage::from_bytes(bytes.as_ref()); + assert_eq!(page_back, page) + } +} diff --git a/src/page/mod.rs b/src/page/mod.rs index cca1aa4..61a4b99 100644 --- a/src/page/mod.rs +++ b/src/page/mod.rs @@ -14,7 +14,10 @@ use crate::{align, SizeMeasurable}; pub use data::DataPage; pub use header::{GeneralHeader, DATA_VERSION}; -pub use index::{get_index_page_size_from_data_length, IndexPage, IndexValue, TableOfContentsPage}; +pub use index::{ + get_index_page_size_from_data_length, IndexPage, IndexPageUtility, IndexValue, + TableOfContentsPage, UnsizedIndexPage, +}; //pub use iterators::{DataIterator, LinksIterator}; pub use space_info::{Interval, SpaceInfoPage}; pub use ty::PageType; From 894fae94f0e7e12f18345ec3f8732a630644a289 Mon Sep 17 00:00:00 2001 From: Handy-caT <37216852+Handy-caT@users.noreply.github.com> Date: Sat, 5 Apr 2025 13:00:10 +0300 Subject: [PATCH 2/8] WIP --- codegen/src/persistable/generator/mod.rs | 1 + codegen/src/persistable/generator/obj_impl.rs | 195 ++++++++------- .../persistable/generator/persistable_impl.rs | 231 ++++++++++++------ codegen/src/persistable/parser.rs | 9 +- src/page/index/page_for_unsized.rs | 68 ++++-- src/util/sized.rs | 17 ++ 6 files changed, 337 insertions(+), 184 deletions(-) diff --git a/codegen/src/persistable/generator/mod.rs b/codegen/src/persistable/generator/mod.rs index c8b077e..2874fa1 100644 --- a/codegen/src/persistable/generator/mod.rs +++ b/codegen/src/persistable/generator/mod.rs @@ -12,6 +12,7 @@ pub struct Generator { pub struct PersistableAttributes { pub is_full_row: bool, + pub unsized_gens: bool, } impl Generator { diff --git a/codegen/src/persistable/generator/obj_impl.rs b/codegen/src/persistable/generator/obj_impl.rs index 66a4964..d0a4d1a 100644 --- a/codegen/src/persistable/generator/obj_impl.rs +++ b/codegen/src/persistable/generator/obj_impl.rs @@ -4,7 +4,7 @@ use crate::persistable::generator::persistable_impl::is_primitive; use crate::persistable::generator::Generator; use quote::{quote, ToTokens}; -use syn::GenericParam; +use syn::{Field, GenericParam}; impl Generator { pub fn gen_obj_impl_def(&self) -> TokenStream { @@ -55,115 +55,67 @@ impl Generator { } pub fn gen_full_obj_size_fn(&self) -> TokenStream { - let contains_vec = self + let size_fields: Vec<_> = self .struct_def .fields .iter() - .any(|f| f.ty.to_token_stream().to_string().contains("Vec")); - if contains_vec { - let sizes: Vec<_> = self - .struct_def - .fields - .iter() - .map(|f| { - let fn_ident = Ident::new( - format!("{}_size", f.ident.clone().unwrap()).as_str(), - Span::call_site(), - ); - - if f.ty.to_token_stream().to_string().contains("Vec") { - quote! { - Self::#fn_ident(length) - } - } else { - quote! { - Self::#fn_ident() - } - } - }) - .collect(); - quote! { - pub fn persisted_size(length: usize) -> usize { - #(#sizes)+* - } - } - } else { - let sizes: Vec<_> = self - .struct_def - .fields - .iter() - .map(|f| { - let fn_ident = Ident::new( - format!("{}_size", f.ident.clone().unwrap()).as_str(), - Span::call_site(), - ); - - quote! { - Self::#fn_ident() - } - }) - .collect(); - quote! { - pub fn persisted_size() -> usize { - #(#sizes)+* + .enumerate() + .filter(|(_, f)| f.ident.clone().unwrap().to_string().contains("size")) + .map(|(_, f)| { + let ident = f.ident.as_ref().unwrap(); + quote! { + #ident: usize } - } - } - } + }) + .collect(); - fn gen_field_sizes_fns(&self) -> TokenStream { - let field_sizes = self.struct_def.fields.iter().map(|f| { - if f.ty.to_token_stream().to_string().contains("Vec") { - let ty = &f.ty; - let inner_ty_str = ty.to_token_stream().to_string().replace(" ", ""); - let mut inner_ty_str = inner_ty_str.replace("Vec<", ""); - inner_ty_str.pop(); - let inner_ty: TokenStream = inner_ty_str.parse().unwrap(); - let value_fn_ident = Ident::new( - format!("{}_value_size", f.ident.clone().unwrap()).as_str(), - Span::call_site(), - ); + let sizes: Vec<_> = self + .struct_def + .fields + .iter() + .map(|f| { let fn_ident = Ident::new( format!("{}_size", f.ident.clone().unwrap()).as_str(), Span::call_site(), ); - let len_in_vec = if is_primitive(&inner_ty_str) { + let size_ident = if size_fields.len() == 1 { quote! { - align(length * <#inner_ty as Default>::default().aligned_size()) + 8 + size } } else { quote! { - length * align8(<#inner_ty as Default>::default().aligned_size()) + 8 + #fn_ident } }; - let len_value = if is_primitive(&inner_ty_str) { + + if f.ty.to_token_stream().to_string().contains("Vec") + || f.ty.to_token_stream().to_string().contains("String") + { quote! { - <#inner_ty as Default>::default().aligned_size() + Self::#fn_ident(#size_ident) } } else { quote! { - align8(<#inner_ty as Default>::default().aligned_size()) - } - }; - quote! { - pub fn #value_fn_ident() -> usize { - #len_value - } - pub fn #fn_ident(length: usize) -> usize { - #len_in_vec + Self::#fn_ident() } } - } else { - let ident = &f.ty; - let fn_ident = Ident::new( - format!("{}_size", f.ident.clone().unwrap()).as_str(), - Span::call_site(), - ); - quote! { - pub fn #fn_ident() -> usize { - <#ident as Default>::default().aligned_size() - } + }) + .collect(); + quote! { + pub fn persisted_size(#(#size_fields),*) -> usize { + #(#sizes)+* } + } + } + + fn gen_field_sizes_fns(&self) -> TokenStream { + let field_sizes = self.struct_def.fields.iter().map(|f| { + if f.ty.to_token_stream().to_string().contains("Vec") { + self.gen_vec_size_fns(&f) + } else if f.ty.to_token_stream().to_string().contains("String") { + self.gen_string_size_fn(&f) + } else { + self.gen_primitive_size_fn(&f) } }); @@ -171,4 +123,71 @@ impl Generator { #(#field_sizes)* } } + + fn gen_primitive_size_fn(&self, f: &Field) -> TokenStream { + let ty = &f.ty; + let fn_ident = Ident::new( + format!("{}_size", f.ident.clone().unwrap()).as_str(), + Span::call_site(), + ); + quote! { + pub fn #fn_ident() -> usize { + <#ty as Default>::default().aligned_size() + } + } + } + + fn gen_string_size_fn(&self, f: &Field) -> TokenStream { + let fn_ident = Ident::new( + format!("{}_size", f.ident.clone().unwrap()).as_str(), + Span::call_site(), + ); + quote! { + pub fn #fn_ident(length: usize) -> usize { + align(length + 8) + } + } + } + + fn gen_vec_size_fns(&self, f: &Field) -> TokenStream { + let ty = &f.ty; + let inner_ty_str = ty.to_token_stream().to_string().replace(" ", ""); + let mut inner_ty_str = inner_ty_str.replace("Vec<", ""); + inner_ty_str.pop(); + let inner_ty: TokenStream = inner_ty_str.parse().unwrap(); + let value_fn_ident = Ident::new( + format!("{}_value_size", f.ident.clone().unwrap()).as_str(), + Span::call_site(), + ); + let fn_ident = Ident::new( + format!("{}_size", f.ident.clone().unwrap()).as_str(), + Span::call_site(), + ); + let len_in_vec = if is_primitive(&inner_ty_str) { + quote! { + align(length * <#inner_ty as Default>::default().aligned_size()) + 8 + } + } else { + quote! { + length * align8(<#inner_ty as Default>::default().aligned_size()) + 8 + } + }; + let len_value = if is_primitive(&inner_ty_str) { + quote! { + <#inner_ty as Default>::default().aligned_size() + } + } else { + quote! { + align8(<#inner_ty as Default>::default().aligned_size()) + } + }; + quote! { + pub fn #value_fn_ident() -> usize { + #len_value + } + pub fn #fn_ident(length: usize) -> usize { + #len_in_vec + } + } + } } diff --git a/codegen/src/persistable/generator/persistable_impl.rs b/codegen/src/persistable/generator/persistable_impl.rs index a98d33b..9cfaad8 100644 --- a/codegen/src/persistable/generator/persistable_impl.rs +++ b/codegen/src/persistable/generator/persistable_impl.rs @@ -1,8 +1,9 @@ use crate::persistable::generator::Generator; -use proc_macro2::TokenStream; + +use proc_macro2::{Ident, TokenStream}; use quote::{quote, ToTokens}; use syn::spanned::Spanned; -use syn::GenericParam; +use syn::{Field, GenericParam, Type}; pub fn is_primitive(ty: &str) -> bool { matches!( @@ -121,7 +122,7 @@ impl Generator { .collect(); quote! { fn as_bytes(&self) -> impl AsRef<[u8]> { - let mut bytes = Vec::with_capacity(self.size as usize); + let mut bytes = vec![]; #(#field_serialize)* bytes } @@ -129,90 +130,172 @@ impl Generator { } fn gen_perisistable_by_parts_from_bytes_fn(&self) -> syn::Result { - let size_field = self + let size_fields: Vec<_> = self .struct_def .fields .iter() .enumerate() - .find(|(_, f)| f.ident.clone().unwrap() == "size"); - if let Some((pos, size_field)) = size_field { - if pos != 0 { + .filter(|(_, f)| f.ident.clone().unwrap().to_string().contains("size")) + .collect(); + + if size_fields.len() == 1 { + let (pos, size_field) = size_fields.first().unwrap(); + if size_field.ident.as_ref().unwrap() != "size" { + return Err(syn::Error::new( + size_field.span(), + "If single size is defined, it should have name `size`", + )); + } + if *pos != 0 { return Err(syn::Error::new( size_field.span(), "`size` field should be first field in a struct", )); } - let size_type = &size_field.ty; - let field_deserialize: Vec<_> = self - .struct_def - .fields - .iter() - .filter(|f| !f.ident.clone().unwrap().to_string().contains("size")) - .map(|f| { - let ident = &f.ident.clone().expect("is not tuple struct"); - if f.ty.to_token_stream().to_string().contains("Vec") { - let ty = &f.ty; - let inner_ty_str = ty.to_token_stream().to_string().replace(" ", ""); - let mut inner_ty_str = inner_ty_str.replace("Vec<", ""); - inner_ty_str.pop(); - let inner_ty: TokenStream = inner_ty_str.parse().unwrap(); - let len = if is_primitive(&inner_ty_str) { - quote! { - let values_len = align(size as usize * <#inner_ty as Default>::default().aligned_size()) + 8; - } - } else { - quote! { - let values_len = size as usize * align8(<#inner_ty as Default>::default().aligned_size()) + 8; - } - }; - quote! { - #len - let mut v = rkyv::util::AlignedVec::<4>::new(); - v.extend_from_slice(&bytes[offset..offset + values_len]); - let archived = - unsafe { rkyv::access_unchecked::<<#ty as Archive>::Archived>(&v[..]) }; - let #ident = rkyv::deserialize::<#ty, rkyv::rancor::Error>(archived) - .expect("data should be valid"); - offset += values_len; - } - } else { - let ty = &f.ty; - quote! { - let length = #ty::default().aligned_size(); - let mut v = rkyv::util::AlignedVec::<4>::new(); - v.extend_from_slice(&bytes[offset..offset + length]); - let archived = unsafe { rkyv::access_unchecked::<<#ty as Archive>::Archived>(&v[..]) }; - let #ident = rkyv::deserialize::<_, rkyv::rancor::Error>(archived).expect("data should be valid"); - offset += length; - } - } - }) - .collect(); - let fields: Vec<_> = self - .struct_def - .fields - .iter() - .map(|f| f.ident.clone().unwrap()) - .collect(); - - Ok(quote! { - fn from_bytes(bytes: &[u8]) -> Self { - let size_length = #size_type::default().aligned_size(); - let archived = - unsafe { rkyv::access_unchecked::<<#size_type as Archive>::Archived>(&bytes[0..size_length]) }; - let size = - rkyv::deserialize::<#size_type, rkyv::rancor::Error>(archived).expect("data should be valid"); - let mut offset = size_length; - - #(#field_deserialize)* + } else { + let mut correct_order = true; + for i in 0..size_fields.len() { + correct_order = size_fields.iter().any(|(pos, _)| *pos == i) + } + if !correct_order { + return Err(syn::Error::new( + self.struct_def.span(), + "`size_..` fields should be first fields in a struct", + )); + } + } - Self { - #(#fields),* - } + let field_deserialize: Vec<_> = self + .struct_def + .fields + .iter() + .filter(|f| !f.ident.clone().unwrap().to_string().contains("size")) + .map(|f| { + let ident = &f.ident.clone().expect("is not tuple struct"); + if f.ty.to_token_stream().to_string().contains("Vec") { + self.gen_from_bytes_for_vec(&f.ty, ident, &size_fields) + } else if f.ty.to_token_stream().to_string().contains("String") { + self.gen_from_bytes_for_string(&f.ty, ident, &size_fields) + } else { + self.gen_from_bytes_for_primitive(&f.ty, ident) } }) + .collect(); + let fields: Vec<_> = self + .struct_def + .fields + .iter() + .map(|f| f.ident.clone().unwrap()) + .collect(); + + let size_defs: Vec<_> = size_fields.into_iter().map(|(_,f)| { + let size_type = &f.ty; + let size_ident = f.ident.as_ref().unwrap(); + quote! { + let size_length = #size_type::default().aligned_size(); + let archived = + unsafe { rkyv::access_unchecked::<<#size_type as Archive>::Archived>(&bytes[0..size_length]) }; + let #size_ident = + rkyv::deserialize::<#size_type, rkyv::rancor::Error>(archived).expect("data should be valid"); + offset += size_length; + } + }).collect(); + + Ok(quote! { + fn from_bytes(bytes: &[u8]) -> Self { + let mut offset = 0usize; + #(#size_defs)* + + #(#field_deserialize)* + + Self { + #(#fields),* + } + } + }) + } + + fn gen_from_bytes_for_primitive(&self, ty: &Type, ident: &Ident) -> TokenStream { + quote! { + let length = #ty::default().aligned_size(); + let mut v = rkyv::util::AlignedVec::<4>::new(); + v.extend_from_slice(&bytes[offset..offset + length]); + let archived = unsafe { rkyv::access_unchecked::<<#ty as Archive>::Archived>(&v[..]) }; + let #ident = rkyv::deserialize::<_, rkyv::rancor::Error>(archived).expect("data should be valid"); + offset += length; + } + } + + fn gen_from_bytes_for_vec( + &self, + ty: &Type, + ident: &Ident, + size_fields: &Vec<(usize, &Field)>, + ) -> TokenStream { + let inner_ty_str = ty.to_token_stream().to_string().replace(" ", ""); + let mut inner_ty_str = inner_ty_str.replace("Vec<", ""); + inner_ty_str.pop(); + let inner_ty: TokenStream = inner_ty_str.parse().unwrap(); + let size_ident = if size_fields.len() == 1 { + size_fields.first().unwrap().1.ident.as_ref().unwrap() } else { - todo!("Add named size's search"); + let val = size_fields.iter().find(|(_, f)| { + f.ident + .as_ref() + .unwrap() + .to_string() + .contains(ident.to_string().as_str()) + }); + val.unwrap().1.ident.as_ref().unwrap() + }; + let len = if is_primitive(&inner_ty_str) { + quote! { + let values_len = align(#size_ident as usize * <#inner_ty as Default>::default().aligned_size()) + 8; + } + } else { + quote! { + let values_len = #size_ident as usize * align8(<#inner_ty as Default>::default().aligned_size()) + 8; + } + }; + quote! { + #len + let mut v = rkyv::util::AlignedVec::<4>::new(); + v.extend_from_slice(&bytes[offset..offset + values_len]); + let archived = + unsafe { rkyv::access_unchecked::<<#ty as Archive>::Archived>(&v[..]) }; + let #ident = rkyv::deserialize::<#ty, rkyv::rancor::Error>(archived) + .expect("data should be valid"); + offset += values_len; + } + } + + fn gen_from_bytes_for_string( + &self, + ty: &Type, + ident: &Ident, + size_fields: &Vec<(usize, &Field)>, + ) -> TokenStream { + let size_ident = if size_fields.len() == 1 { + size_fields.first().unwrap().1.ident.as_ref().unwrap() + } else { + let val = size_fields.iter().find(|(_, f)| { + f.ident + .as_ref() + .unwrap() + .to_string() + .contains(ident.to_string().as_str()) + }); + val.unwrap().1.ident.as_ref().unwrap() + }; + quote! { + let values_len = align(#size_ident + 8) + let mut v = rkyv::util::AlignedVec::<4>::new(); + v.extend_from_slice(&bytes[offset..offset + values_len]); + let archived = + unsafe { rkyv::access_unchecked::<<#ty as Archive>::Archived>(&v[..]) }; + let #ident = rkyv::deserialize::<#ty, rkyv::rancor::Error>(archived) + .expect("data should be valid"); + offset += values_len; } } } diff --git a/codegen/src/persistable/parser.rs b/codegen/src/persistable/parser.rs index 061df89..7e63517 100644 --- a/codegen/src/persistable/parser.rs +++ b/codegen/src/persistable/parser.rs @@ -15,7 +15,10 @@ impl Parser { } pub fn parse_attributes(attrs: &Vec) -> PersistableAttributes { - let mut res = PersistableAttributes { is_full_row: true }; + let mut res = PersistableAttributes { + is_full_row: true, + unsized_gens: false, + }; for attr in attrs { if attr.path().to_token_stream().to_string().as_str() == "persistable" { @@ -24,6 +27,10 @@ impl Parser { res.is_full_row = false; return Ok(()); } + if meta.path.is_ident("unsized_gens") { + res.unsized_gens = true; + return Ok(()); + } Ok(()) }) .expect("always ok even on unrecognized attrs"); diff --git a/src/page/index/page_for_unsized.rs b/src/page/index/page_for_unsized.rs index 3d67f2f..697a172 100644 --- a/src/page/index/page_for_unsized.rs +++ b/src/page/index/page_for_unsized.rs @@ -18,7 +18,8 @@ use crate::{seek_to_page_start, IndexValue, SizeMeasurable, GENERAL_HEADER_SIZE} #[derive(Archive, Clone, Deserialize, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] pub struct UnsizedIndexPage { - pub size: u16, + pub slots_size: u16, + pub node_id_size: u16, pub node_id: T, pub last_value_offset: u32, pub slots: Vec<(u32, u16)>, @@ -30,7 +31,8 @@ pub struct UnsizedIndexPage )] #[persistable(by_parts)] pub struct UnsizedIndexPageUtility { - pub size: u16, + pub slots_size: u16, + pub node_id_size: u16, pub node_id: T, pub last_value_offset: u32, pub slots: Vec<(u32, u16)>, @@ -55,19 +57,32 @@ where let offset = GENERAL_HEADER_SIZE as i64; file.seek(SeekFrom::Current(offset)).await?; - let mut size_bytes = vec![0u8; UnsizedIndexPageUtility::::size_size()]; - file.read_exact(size_bytes.as_mut_slice()).await?; + let mut slot_size_bytes = vec![0u8; UnsizedIndexPageUtility::::slots_size_size()]; + file.read_exact(slot_size_bytes.as_mut_slice()).await?; let archived = unsafe { rkyv::access_unchecked::<::Archived>( - &size_bytes[0..UnsizedIndexPageUtility::::size_size()], + &slot_size_bytes[0..UnsizedIndexPageUtility::::slots_size_size()], ) }; - let size = + let slots_size = + rkyv::deserialize::(archived).expect("data should be valid"); + let mut node_id_size_bytes = vec![0u8; UnsizedIndexPageUtility::::node_id_size_size()]; + file.read_exact(node_id_size_bytes.as_mut_slice()).await?; + let archived = unsafe { + rkyv::access_unchecked::<::Archived>( + &node_id_size_bytes[0..UnsizedIndexPageUtility::::node_id_size_size()], + ) + }; + let node_id_size = rkyv::deserialize::(archived).expect("data should be valid"); - let index_utility_len = UnsizedIndexPageUtility::::persisted_size(size as usize); + let index_utility_len = UnsizedIndexPageUtility::::persisted_size( + slots_size as usize, + node_id_size as usize, + ); file.seek(SeekFrom::Current( - -(UnsizedIndexPageUtility::::size_size() as i64), + -(UnsizedIndexPageUtility::::slots_size_size() as i64 + + UnsizedIndexPageUtility::::node_id_size_size() as i64), )) .await?; let mut index_utility_bytes = vec![0u8; index_utility_len]; @@ -92,7 +107,8 @@ where let bytes = rkyv::to_bytes::(&value)?; let len = bytes.len() as u32; Ok(Self { - size: 1, + slots_size: 1, + node_id_size: node_id.aligned_size() as u16, node_id, last_value_offset: len, slots: vec![(len, len as u16)], @@ -140,7 +156,8 @@ where fn as_bytes(&self) -> impl AsRef<[u8]> + Send { let data_length = DATA_LENGTH as usize; let utility = UnsizedIndexPageUtility { - size: self.size, + slots_size: self.slots_size, + node_id_size: self.node_id_size, node_id: self.node_id.clone(), last_value_offset: self.last_value_offset, slots: self.slots.clone(), @@ -162,15 +179,22 @@ where } fn from_bytes(bytes: &[u8]) -> Self { - let size_bytes = &bytes[0..UnsizedIndexPageUtility::::size_size()]; - let archived = unsafe { - rkyv::access_unchecked::<::Archived>( - &size_bytes[0..UnsizedIndexPageUtility::::size_size()], - ) - }; - let size = + let slots_size_bytes = &bytes[0..UnsizedIndexPageUtility::::slots_size_size()]; + let archived = + unsafe { rkyv::access_unchecked::<::Archived>(&slots_size_bytes) }; + let slots_size = + rkyv::deserialize::(archived).expect("data should be valid"); + let node_id_size_bytes = &bytes[UnsizedIndexPageUtility::::slots_size_size() + ..UnsizedIndexPageUtility::::node_id_size_size() + + UnsizedIndexPageUtility::::node_id_size_size()]; + let archived = + unsafe { rkyv::access_unchecked::<::Archived>(&node_id_size_bytes) }; + let node_id_size = rkyv::deserialize::(archived).expect("data should be valid"); - let utility_len = UnsizedIndexPageUtility::::persisted_size(size as usize); + let utility_len = UnsizedIndexPageUtility::::persisted_size( + slots_size as usize, + node_id_size as usize, + ); let utility = UnsizedIndexPageUtility::::from_bytes(&bytes[0..utility_len]); let mut index_values = Vec::with_capacity(utility.slots.len()); for (offset, len) in &utility.slots { @@ -186,7 +210,8 @@ where } Self { - size, + slots_size, + node_id_size, node_id: utility.node_id, last_value_offset: utility.last_value_offset, slots: utility.slots, @@ -202,14 +227,15 @@ mod test { #[test] fn to_bytes_and_back() { let value = IndexValue { - key: "SomeID".to_string(), + key: "Something for Someone".to_string(), link: Link { page_id: 0.into(), offset: 0, length: 40, }, }; - let page = UnsizedIndexPage::<_, 1024>::new("SomeID".to_string(), value).unwrap(); + let page = + UnsizedIndexPage::<_, 1024>::new("Something for Someone".to_string(), value).unwrap(); let bytes = page.as_bytes(); assert_eq!(bytes.as_ref().len(), 1024); let page_back = UnsizedIndexPage::from_bytes(bytes.as_ref()); diff --git a/src/util/sized.rs b/src/util/sized.rs index 3e79f2a..1477e28 100644 --- a/src/util/sized.rs +++ b/src/util/sized.rs @@ -135,6 +135,23 @@ where } } +/// Marks an objects that can return theirs approximate size after archiving via +/// [`rkyv`]. +pub trait VariableSizeMeasurable { + /// Returns approximate size of the object archiving via [`rkyv`]. + fn aligned_size(length: usize) -> usize; +} + +impl VariableSizeMeasurable for String { + fn aligned_size(length: usize) -> usize { + if length <= 8 { + 8 + } else { + align(length + 8) + } + } +} + #[cfg(test)] mod test { use crate::util::sized::SizeMeasurable; From 819ea085e88b2f111440c2aadad74c54626af18e Mon Sep 17 00:00:00 2001 From: Handy-caT <37216852+Handy-caT@users.noreply.github.com> Date: Mon, 7 Apr 2025 15:35:24 +0300 Subject: [PATCH 3/8] fix unsized page logic --- codegen/src/persistable/generator/mod.rs | 3 +- codegen/src/persistable/generator/obj_impl.rs | 55 +++++++++++++++++-- .../persistable/generator/persistable_impl.rs | 52 +++++++++++++++++- codegen/src/persistable/mod.rs | 1 + codegen/src/persistable/parser.rs | 4 +- src/lib.rs | 2 +- src/page/index/page_for_unsized.rs | 21 ++++--- src/util/mod.rs | 2 +- 8 files changed, 119 insertions(+), 21 deletions(-) diff --git a/codegen/src/persistable/generator/mod.rs b/codegen/src/persistable/generator/mod.rs index 2874fa1..51afb5b 100644 --- a/codegen/src/persistable/generator/mod.rs +++ b/codegen/src/persistable/generator/mod.rs @@ -7,12 +7,13 @@ use syn::ItemStruct; pub struct Generator { pub is_full_row: bool, + pub is_generic_unsized: bool, pub struct_def: ItemStruct, } pub struct PersistableAttributes { pub is_full_row: bool, - pub unsized_gens: bool, + pub is_generic_unsized: bool, } impl Generator { diff --git a/codegen/src/persistable/generator/obj_impl.rs b/codegen/src/persistable/generator/obj_impl.rs index d0a4d1a..abef4cd 100644 --- a/codegen/src/persistable/generator/obj_impl.rs +++ b/codegen/src/persistable/generator/obj_impl.rs @@ -68,6 +68,19 @@ impl Generator { } }) .collect(); + let gens: Vec<_> = self + .struct_def + .generics + .params + .iter() + .filter_map(|p| { + if let GenericParam::Type(t) = p { + Some(t.ident.to_string()) + } else { + None + } + }) + .collect(); let sizes: Vec<_> = self .struct_def @@ -88,12 +101,15 @@ impl Generator { } }; - if f.ty.to_token_stream().to_string().contains("Vec") - || f.ty.to_token_stream().to_string().contains("String") - { + let field_type_str = f.ty.to_token_stream().to_string(); + if field_type_str.contains("Vec") || field_type_str.contains("String") { quote! { Self::#fn_ident(#size_ident) } + } else if gens.contains(&field_type_str) && self.is_generic_unsized { + quote! { + #size_ident + } } else { quote! { Self::#fn_ident() @@ -109,11 +125,27 @@ impl Generator { } fn gen_field_sizes_fns(&self) -> TokenStream { + let gens: Vec<_> = self + .struct_def + .generics + .params + .iter() + .filter_map(|p| { + if let GenericParam::Type(t) = p { + Some(t.ident.to_string()) + } else { + None + } + }) + .collect(); let field_sizes = self.struct_def.fields.iter().map(|f| { - if f.ty.to_token_stream().to_string().contains("Vec") { + let field_type_str = f.ty.to_token_stream().to_string(); + if field_type_str.contains("Vec") { self.gen_vec_size_fns(&f) - } else if f.ty.to_token_stream().to_string().contains("String") { + } else if field_type_str.contains("String") { self.gen_string_size_fn(&f) + } else if gens.contains(&field_type_str) && self.is_generic_unsized { + self.gen_generic_size_fn(&f) } else { self.gen_primitive_size_fn(&f) } @@ -149,6 +181,19 @@ impl Generator { } } + fn gen_generic_size_fn(&self, f: &Field) -> TokenStream { + let fn_ident = Ident::new( + format!("{}_size", f.ident.clone().unwrap()).as_str(), + Span::call_site(), + ); + let ty_ = &f.ty; + quote! { + pub fn #fn_ident(length: usize) -> usize { + <#ty_ as VariableSizeMeasurable>::aligned_size(length) + } + } + } + fn gen_vec_size_fns(&self, f: &Field) -> TokenStream { let ty = &f.ty; let inner_ty_str = ty.to_token_stream().to_string().replace(" ", ""); diff --git a/codegen/src/persistable/generator/persistable_impl.rs b/codegen/src/persistable/generator/persistable_impl.rs index 9cfaad8..ee588e1 100644 --- a/codegen/src/persistable/generator/persistable_impl.rs +++ b/codegen/src/persistable/generator/persistable_impl.rs @@ -137,6 +137,19 @@ impl Generator { .enumerate() .filter(|(_, f)| f.ident.clone().unwrap().to_string().contains("size")) .collect(); + let gens: Vec<_> = self + .struct_def + .generics + .params + .iter() + .filter_map(|p| { + if let GenericParam::Type(t) = p { + Some(t.ident.to_string()) + } else { + None + } + }) + .collect(); if size_fields.len() == 1 { let (pos, size_field) = size_fields.first().unwrap(); @@ -172,10 +185,13 @@ impl Generator { .filter(|f| !f.ident.clone().unwrap().to_string().contains("size")) .map(|f| { let ident = &f.ident.clone().expect("is not tuple struct"); - if f.ty.to_token_stream().to_string().contains("Vec") { + let field_type_str = f.ty.to_token_stream().to_string(); + if field_type_str.contains("Vec") { self.gen_from_bytes_for_vec(&f.ty, ident, &size_fields) - } else if f.ty.to_token_stream().to_string().contains("String") { + } else if field_type_str.contains("String") { self.gen_from_bytes_for_string(&f.ty, ident, &size_fields) + } else if gens.contains(&field_type_str) && self.is_generic_unsized { + self.gen_from_bytes_for_unsized_generic(&f.ty, ident, &size_fields) } else { self.gen_from_bytes_for_primitive(&f.ty, ident) } @@ -194,7 +210,7 @@ impl Generator { quote! { let size_length = #size_type::default().aligned_size(); let archived = - unsafe { rkyv::access_unchecked::<<#size_type as Archive>::Archived>(&bytes[0..size_length]) }; + unsafe { rkyv::access_unchecked::<<#size_type as Archive>::Archived>(&bytes[offset..offset + size_length]) }; let #size_ident = rkyv::deserialize::<#size_type, rkyv::rancor::Error>(archived).expect("data should be valid"); offset += size_length; @@ -298,4 +314,34 @@ impl Generator { offset += values_len; } } + + fn gen_from_bytes_for_unsized_generic( + &self, + ty: &Type, + ident: &Ident, + size_fields: &Vec<(usize, &Field)>, + ) -> TokenStream { + let size_ident = if size_fields.len() == 1 { + size_fields.first().unwrap().1.ident.as_ref().unwrap() + } else { + let val = size_fields.iter().find(|(_, f)| { + f.ident + .as_ref() + .unwrap() + .to_string() + .contains(ident.to_string().as_str()) + }); + val.unwrap().1.ident.as_ref().unwrap() + }; + quote! { + let values_len = #size_ident as usize; + let mut v = rkyv::util::AlignedVec::<4>::new(); + v.extend_from_slice(&bytes[offset..offset + values_len]); + let archived = + unsafe { rkyv::access_unchecked::<<#ty as Archive>::Archived>(&v[..]) }; + let #ident = rkyv::deserialize::<#ty, rkyv::rancor::Error>(archived) + .expect("data should be valid"); + offset += values_len; + } + } } diff --git a/codegen/src/persistable/mod.rs b/codegen/src/persistable/mod.rs index afd2812..85fcbdc 100644 --- a/codegen/src/persistable/mod.rs +++ b/codegen/src/persistable/mod.rs @@ -10,6 +10,7 @@ pub fn expand(input: &TokenStream) -> syn::Result { let attrs = Parser::parse_attributes(&input_struct.attrs); let gen = generator::Generator { is_full_row: attrs.is_full_row, + is_generic_unsized: attrs.is_generic_unsized, struct_def: input_struct, }; diff --git a/codegen/src/persistable/parser.rs b/codegen/src/persistable/parser.rs index 7e63517..ba567ca 100644 --- a/codegen/src/persistable/parser.rs +++ b/codegen/src/persistable/parser.rs @@ -17,7 +17,7 @@ impl Parser { pub fn parse_attributes(attrs: &Vec) -> PersistableAttributes { let mut res = PersistableAttributes { is_full_row: true, - unsized_gens: false, + is_generic_unsized: false, }; for attr in attrs { @@ -28,7 +28,7 @@ impl Parser { return Ok(()); } if meta.path.is_ident("unsized_gens") { - res.unsized_gens = true; + res.is_generic_unsized = true; return Ok(()); } Ok(()) diff --git a/src/lib.rs b/src/lib.rs index 978ab14..eee2b81 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,4 +18,4 @@ pub use page::{ }; pub use persistence::{PersistableIndex, PersistableTable}; pub use space::Id as SpaceId; -pub use util::{align, align8, align_vec, Persistable, SizeMeasurable}; +pub use util::{align, align8, align_vec, Persistable, SizeMeasurable, VariableSizeMeasurable}; diff --git a/src/page/index/page_for_unsized.rs b/src/page/index/page_for_unsized.rs index 697a172..fd45a1f 100644 --- a/src/page/index/page_for_unsized.rs +++ b/src/page/index/page_for_unsized.rs @@ -10,14 +10,17 @@ use std::io::SeekFrom; use tokio::fs::File; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; -use crate::align8; use crate::page::index::IndexPageUtility; use crate::page::PageId; use crate::Persistable; +use crate::{align8, VariableSizeMeasurable}; use crate::{seek_to_page_start, IndexValue, SizeMeasurable, GENERAL_HEADER_SIZE}; #[derive(Archive, Clone, Deserialize, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] -pub struct UnsizedIndexPage { +pub struct UnsizedIndexPage< + T: Default + SizeMeasurable + VariableSizeMeasurable, + const DATA_LENGTH: u32, +> { pub slots_size: u16, pub node_id_size: u16, pub node_id: T, @@ -29,8 +32,8 @@ pub struct UnsizedIndexPage #[derive( Archive, Clone, Deserialize, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize, Persistable, )] -#[persistable(by_parts)] -pub struct UnsizedIndexPageUtility { +#[persistable(by_parts, unsized_gens)] +pub struct UnsizedIndexPageUtility { pub slots_size: u16, pub node_id_size: u16, pub node_id: T, @@ -38,8 +41,8 @@ pub struct UnsizedIndexPageUtility { pub slots: Vec<(u32, u16)>, } -impl IndexPageUtility - for UnsizedIndexPage +impl + IndexPageUtility for UnsizedIndexPage where T: Archive + for<'a> Serialize< @@ -98,6 +101,7 @@ where T: Archive + Default + SizeMeasurable + + VariableSizeMeasurable + for<'a> Serialize< Strategy, Share>, rkyv::rancor::Error>, >, @@ -148,6 +152,7 @@ where + Clone + Default + SizeMeasurable + + VariableSizeMeasurable + for<'a> Serialize< Strategy, Share>, rkyv::rancor::Error>, >, @@ -227,7 +232,7 @@ mod test { #[test] fn to_bytes_and_back() { let value = IndexValue { - key: "Something for Someone".to_string(), + key: "Someone from somewhere".to_string(), link: Link { page_id: 0.into(), offset: 0, @@ -235,7 +240,7 @@ mod test { }, }; let page = - UnsizedIndexPage::<_, 1024>::new("Something for Someone".to_string(), value).unwrap(); + UnsizedIndexPage::<_, 1024>::new("Someone from somewhere".to_string(), value).unwrap(); let bytes = page.as_bytes(); assert_eq!(bytes.as_ref().len(), 1024); let page_back = UnsizedIndexPage::from_bytes(bytes.as_ref()); diff --git a/src/util/mod.rs b/src/util/mod.rs index 237a16d..84a1a24 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -2,4 +2,4 @@ mod persistable; mod sized; pub use persistable::Persistable; -pub use sized::{align, align8, align_vec, SizeMeasurable}; +pub use sized::{align, align8, align_vec, SizeMeasurable, VariableSizeMeasurable}; From 31c38420ac203e72131ddab59e6bf8942e5c17ae Mon Sep 17 00:00:00 2001 From: Handy-caT <37216852+Handy-caT@users.noreply.github.com> Date: Tue, 8 Apr 2025 19:05:17 +0300 Subject: [PATCH 4/8] add split logic for the page --- src/page/index/page_for_unsized.rs | 129 +++++++++++++++++++++++++++-- 1 file changed, 124 insertions(+), 5 deletions(-) diff --git a/src/page/index/page_for_unsized.rs b/src/page/index/page_for_unsized.rs index fd45a1f..9506c2c 100644 --- a/src/page/index/page_for_unsized.rs +++ b/src/page/index/page_for_unsized.rs @@ -7,14 +7,15 @@ use rkyv::ser::Serializer; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize}; use std::io::SeekFrom; +use std::mem; use tokio::fs::File; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; use crate::page::index::IndexPageUtility; use crate::page::PageId; -use crate::Persistable; use crate::{align8, VariableSizeMeasurable}; use crate::{seek_to_page_start, IndexValue, SizeMeasurable, GENERAL_HEADER_SIZE}; +use crate::{IndexPage, Persistable}; #[derive(Archive, Clone, Deserialize, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] pub struct UnsizedIndexPage< @@ -41,6 +42,15 @@ pub struct UnsizedIndexPageUtility, } +impl UnsizedIndexPageUtility { + pub fn update_node_id(&mut self, node_id: T) -> eyre::Result<()> { + self.node_id_size = node_id.aligned_size() as u16; + self.node_id = node_id; + + Ok(()) + } +} + impl IndexPageUtility for UnsizedIndexPage where @@ -108,8 +118,7 @@ where ::Archived: Deserialize>, { pub fn new(node_id: T, value: IndexValue) -> eyre::Result { - let bytes = rkyv::to_bytes::(&value)?; - let len = bytes.len() as u32; + let len = value.aligned_size() as u32; Ok(Self { slots_size: 1, node_id_size: node_id.aligned_size() as u16, @@ -120,6 +129,54 @@ where }) } + fn new_with_values(values: Vec>) -> Self + where + T: Clone, + { + let slots_size = values.len() as u16; + let node_id = values.last().expect("Node should be not empty").key.clone(); + let node_id_size = node_id.aligned_size() as u16; + let mut last_value_offset = 0; + let mut slots = vec![]; + for val in &values { + let len = val.aligned_size() as u32; + slots.push((last_value_offset, len as u16)); + last_value_offset += len; + } + Self { + slots_size, + node_id_size, + node_id, + last_value_offset, + slots, + index_values: values, + } + } + + fn rebuild(&mut self) { + self.node_id_size = self.node_id.aligned_size() as u16; + self.last_value_offset = 0; + let mut slots = vec![]; + for val in &self.index_values { + let len = val.aligned_size() as u32; + slots.push((self.last_value_offset, len as u16)); + self.last_value_offset += len; + } + self.slots = slots; + self.slots_size = self.slots.len() as u16 + } + + pub fn split(&mut self, index: usize) -> UnsizedIndexPage + where + T: Clone, + { + let index_values = self.index_values.split_off(index); + let mut new_page = UnsizedIndexPage::new_with_values(index_values); + self.rebuild(); + + new_page + } + pub async fn persist_value( file: &mut File, page_id: PageId, @@ -139,11 +196,40 @@ where let bytes = rkyv::to_bytes::(&value)?; let offset = current_offset + bytes.len() as u32; - file.seek(SeekFrom::Current(offset as i64)).await?; + file.seek(SeekFrom::Current(-(offset as i64))).await?; file.write_all(bytes.as_slice()).await?; Ok(offset) } + + async fn read_value(file: &mut File, len: u16) -> eyre::Result> + where + T: Archive, + ::Archived: Deserialize>, + { + let mut bytes = vec![0u8; len as usize]; + file.read_exact(bytes.as_mut_slice()).await?; + let mut v = AlignedVec::<4>::new(); + v.extend_from_slice(bytes.as_slice()); + let archived = + unsafe { rkyv::access_unchecked::< as Archive>::Archived>(&v[..]) }; + Ok(rkyv::deserialize(archived).expect("data should be valid")) + } + + pub async fn read_value_with_offset( + file: &mut File, + page_id: PageId, + offset: u32, + len: u16, + ) -> eyre::Result> + where + T: Archive, + ::Archived: Deserialize>, + { + seek_to_page_start(file, page_id.0 + 1).await?; + file.seek(SeekFrom::Current(-(offset as i64))).await?; + Self::read_value(file, len).await + } } impl Persistable for UnsizedIndexPage @@ -227,7 +313,7 @@ where #[cfg(test)] mod test { - use crate::{IndexValue, Link, Persistable, UnsizedIndexPage}; + use crate::{IndexPage, IndexValue, Link, Persistable, UnsizedIndexPage}; #[test] fn to_bytes_and_back() { @@ -246,4 +332,37 @@ mod test { let page_back = UnsizedIndexPage::from_bytes(bytes.as_ref()); assert_eq!(page_back, page) } + + #[test] + fn split() { + let mut values = vec![]; + for i in 0..10 { + values.push(IndexValue { + key: format!("{}___________________{}", i, i), + link: Link { + page_id: 0.into(), + offset: i * 24, + length: 24, + }, + }) + } + let mut page = UnsizedIndexPage::::new_with_values(values); + let split = page.split(5); + + assert_eq!(page.slots_size, 5); + let offset = page.slots.iter().map(|(_, l)| *l).sum::(); + assert_eq!(page.last_value_offset, offset as u32); + assert_eq!( + page.last_value_offset, + page.slots.last().unwrap().0 + page.slots.last().unwrap().1 as u32 + ); + + assert_eq!(split.slots_size, 5); + let offset = split.slots.iter().map(|(_, l)| *l).sum::(); + assert_eq!(split.last_value_offset, offset as u32); + assert_eq!( + split.last_value_offset, + page.slots.last().unwrap().0 + page.slots.last().unwrap().1 as u32 + ); + } } From 74db64cab9bba2239db01aa85f1e1dc086676143 Mon Sep 17 00:00:00 2001 From: Handy-caT <37216852+Handy-caT@users.noreply.github.com> Date: Tue, 8 Apr 2025 19:13:05 +0300 Subject: [PATCH 5/8] correction --- src/page/index/page_for_unsized.rs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/page/index/page_for_unsized.rs b/src/page/index/page_for_unsized.rs index 9506c2c..235769b 100644 --- a/src/page/index/page_for_unsized.rs +++ b/src/page/index/page_for_unsized.rs @@ -140,8 +140,8 @@ where let mut slots = vec![]; for val in &values { let len = val.aligned_size() as u32; - slots.push((last_value_offset, len as u16)); last_value_offset += len; + slots.push((last_value_offset, len as u16)); } Self { slots_size, @@ -159,8 +159,8 @@ where let mut slots = vec![]; for val in &self.index_values { let len = val.aligned_size() as u32; - slots.push((self.last_value_offset, len as u16)); self.last_value_offset += len; + slots.push((self.last_value_offset, len as u16)); } self.slots = slots; self.slots_size = self.slots.len() as u16 @@ -313,7 +313,7 @@ where #[cfg(test)] mod test { - use crate::{IndexPage, IndexValue, Link, Persistable, UnsizedIndexPage}; + use crate::{IndexValue, Link, Persistable, UnsizedIndexPage}; #[test] fn to_bytes_and_back() { @@ -352,17 +352,11 @@ mod test { assert_eq!(page.slots_size, 5); let offset = page.slots.iter().map(|(_, l)| *l).sum::(); assert_eq!(page.last_value_offset, offset as u32); - assert_eq!( - page.last_value_offset, - page.slots.last().unwrap().0 + page.slots.last().unwrap().1 as u32 - ); + assert_eq!(page.last_value_offset, page.slots.last().unwrap().0); assert_eq!(split.slots_size, 5); let offset = split.slots.iter().map(|(_, l)| *l).sum::(); assert_eq!(split.last_value_offset, offset as u32); - assert_eq!( - split.last_value_offset, - page.slots.last().unwrap().0 + page.slots.last().unwrap().1 as u32 - ); + assert_eq!(split.last_value_offset, page.slots.last().unwrap().0) } } From 81dd05b9a7c51c49412062048c05c37005d81bdb Mon Sep 17 00:00:00 2001 From: Handy-caT <37216852+Handy-caT@users.noreply.github.com> Date: Mon, 14 Apr 2025 13:58:41 +0300 Subject: [PATCH 6/8] add VariableSizeMeasure and corrections --- Cargo.toml | 3 +- codegen/src/lib.rs | 8 +++++ .../src/variable_size_measure/generator.rs | 33 +++++++++++++++++ codegen/src/variable_size_measure/mod.rs | 21 +++++++++++ codegen/src/variable_size_measure/parser.rs | 14 ++++++++ src/lib.rs | 6 ++-- src/page/index/mod.rs | 24 +++++++------ src/page/index/page.rs | 3 +- src/page/index/page_for_unsized.rs | 36 ++++++++++++++++--- src/page/mod.rs | 2 +- src/page/ty.rs | 2 ++ src/util/sized.rs | 36 +++++++++++++++++++ 12 files changed, 166 insertions(+), 22 deletions(-) create mode 100644 codegen/src/variable_size_measure/generator.rs create mode 100644 codegen/src/variable_size_measure/mod.rs create mode 100644 codegen/src/variable_size_measure/parser.rs diff --git a/Cargo.toml b/Cargo.toml index bd0f79d..44ca367 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,5 +18,6 @@ derive_more = { version = "1.0.0", features = ["from", "error", "display", "into rkyv = { version = "0.8.9", features = ["uuid-1"] } lockfree = "0.5.1" uuid = { version = "1.11.0", features = ["v4"] } -indexset = { version = "0.11.2", features = ["concurrent", "cdc", "multimap"] } +# indexset = { version = "0.11.2", features = ["concurrent", "cdc", "multimap"] } +indexset = { path = "../indexset", version = "0.11.3", features = ["concurrent", "cdc", "multimap"] } tokio = { version = "1", features = ["full"] } diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs index eb590c9..a8d9986 100644 --- a/codegen/src/lib.rs +++ b/codegen/src/lib.rs @@ -1,5 +1,6 @@ mod persistable; mod size_measure; +mod variable_size_measure; use proc_macro::TokenStream; @@ -10,6 +11,13 @@ pub fn size_measure(input: TokenStream) -> TokenStream { .into() } +#[proc_macro_derive(VariableSizeMeasure)] +pub fn variable_size_measure(input: TokenStream) -> TokenStream { + variable_size_measure::expand(&(input.into())) + .unwrap_or_else(|e| e.to_compile_error()) + .into() +} + #[proc_macro_derive(Persistable, attributes(persistable))] pub fn persistable(input: TokenStream) -> TokenStream { persistable::expand(&(input.into())) diff --git a/codegen/src/variable_size_measure/generator.rs b/codegen/src/variable_size_measure/generator.rs new file mode 100644 index 0000000..bf555ef --- /dev/null +++ b/codegen/src/variable_size_measure/generator.rs @@ -0,0 +1,33 @@ +use proc_macro2::TokenStream; +use quote::quote; +use syn::ItemStruct; + +pub struct Generator { + pub struct_def: ItemStruct, +} + +impl Generator { + pub fn gen_impl(&self) -> TokenStream { + let struct_ident = &self.struct_def.ident; + + let sum = self + .struct_def + .fields + .iter() + .map(|f| { + let ty = &f.ty; + quote! { + <#ty as VariableSizeMeasurable>::aligned_size(length) + } + }) + .collect::>(); + + quote! { + impl VariableSizeMeasurable for #struct_ident { + fn aligned_size(length: usize) -> usize { + align(#(#sum)+*) + } + } + } + } +} diff --git a/codegen/src/variable_size_measure/mod.rs b/codegen/src/variable_size_measure/mod.rs new file mode 100644 index 0000000..40fbfeb --- /dev/null +++ b/codegen/src/variable_size_measure/mod.rs @@ -0,0 +1,21 @@ +mod generator; +mod parser; + +use proc_macro2::TokenStream; +use quote::quote; + +use crate::variable_size_measure::generator::Generator; +use crate::variable_size_measure::parser::Parser; + +pub fn expand(input: &TokenStream) -> syn::Result { + let input_fn = Parser::parse_struct(input)?; + let gen = Generator { + struct_def: input_fn, + }; + + let impl_def = gen.gen_impl(); + + Ok(quote! { + #impl_def + }) +} diff --git a/codegen/src/variable_size_measure/parser.rs b/codegen/src/variable_size_measure/parser.rs new file mode 100644 index 0000000..8649504 --- /dev/null +++ b/codegen/src/variable_size_measure/parser.rs @@ -0,0 +1,14 @@ +use proc_macro2::TokenStream; +use syn::spanned::Spanned; +use syn::ItemStruct; + +pub struct Parser; + +impl Parser { + pub fn parse_struct(input: &TokenStream) -> syn::Result { + match syn::parse2::(input.clone()) { + Ok(data) => Ok(data), + Err(err) => Err(syn::Error::new(input.span(), err.to_string())), + } + } +} diff --git a/src/lib.rs b/src/lib.rs index eee2b81..1fe80fc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,13 +8,13 @@ pub mod util; pub use link::Link; -pub use data_bucket_codegen::SizeMeasure; +pub use data_bucket_codegen::{SizeMeasure, VariableSizeMeasure}; pub use page::{ get_index_page_size_from_data_length, map_data_pages_to_general, parse_data_page, parse_general_header_by_index, parse_page, persist_page, seek_by_link, seek_to_page_start, update_at, DataPage, GeneralHeader, GeneralPage, IndexPage, IndexPageUtility, IndexValue, - Interval, PageType, SpaceInfoPage, TableOfContentsPage, UnsizedIndexPage, DATA_VERSION, - GENERAL_HEADER_SIZE, INNER_PAGE_SIZE, PAGE_SIZE, + Interval, PageType, SpaceInfoPage, TableOfContentsPage, UnsizedIndexPage, + UnsizedIndexPageUtility, DATA_VERSION, GENERAL_HEADER_SIZE, INNER_PAGE_SIZE, PAGE_SIZE, }; pub use persistence::{PersistableIndex, PersistableTable}; pub use space::Id as SpaceId; diff --git a/src/page/index/mod.rs b/src/page/index/mod.rs index f08e622..4c7aaf5 100644 --- a/src/page/index/mod.rs +++ b/src/page/index/mod.rs @@ -15,27 +15,29 @@ mod table_of_contents_page; use crate::page::PageId; pub use page::{get_index_page_size_from_data_length, IndexPage}; -pub use page_for_unsized::UnsizedIndexPage; +pub use page_for_unsized::{UnsizedIndexPage, UnsizedIndexPageUtility}; pub use table_of_contents_page::TableOfContentsPage; pub trait IndexPageUtility { - type Utility: Persistable; + type Utility: Persistable + Send + Sync; - async fn parse_index_page_utility( + fn parse_index_page_utility( file: &mut File, page_id: PageId, - ) -> eyre::Result; + ) -> impl std::future::Future> + Send; - async fn persist_index_page_utility( + fn persist_index_page_utility( file: &mut File, page_id: PageId, utility: Self::Utility, - ) -> eyre::Result<()> { - seek_to_page_start(file, page_id.0).await?; - file.seek(SeekFrom::Current(GENERAL_HEADER_SIZE as i64)) - .await?; - file.write_all(utility.as_bytes().as_ref()).await?; - Ok(()) + ) -> impl std::future::Future> + Send { + async move { + seek_to_page_start(file, page_id.0).await?; + file.seek(SeekFrom::Current(GENERAL_HEADER_SIZE as i64)) + .await?; + file.write_all(utility.as_bytes().as_ref()).await?; + Ok(()) + } } } diff --git a/src/page/index/page.rs b/src/page/index/page.rs index 12c8129..1d068b9 100644 --- a/src/page/index/page.rs +++ b/src/page/index/page.rs @@ -77,7 +77,8 @@ where T: Archive + for<'a> Serialize< Strategy, Share>, rkyv::rancor::Error>, - >, + > + Send + + Sync, ::Archived: Deserialize>, { type Utility = SizedIndexPageUtility; diff --git a/src/page/index/page_for_unsized.rs b/src/page/index/page_for_unsized.rs index 235769b..3599df2 100644 --- a/src/page/index/page_for_unsized.rs +++ b/src/page/index/page_for_unsized.rs @@ -1,4 +1,5 @@ use data_bucket_codegen::Persistable; +use indexset::core::pair::Pair; use rkyv::de::Pool; use rkyv::rancor::Strategy; use rkyv::ser::allocator::ArenaHandle; @@ -7,7 +8,6 @@ use rkyv::ser::Serializer; use rkyv::util::AlignedVec; use rkyv::{Archive, Deserialize, Serialize}; use std::io::SeekFrom; -use std::mem; use tokio::fs::File; use tokio::io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}; @@ -15,7 +15,7 @@ use crate::page::index::IndexPageUtility; use crate::page::PageId; use crate::{align8, VariableSizeMeasurable}; use crate::{seek_to_page_start, IndexValue, SizeMeasurable, GENERAL_HEADER_SIZE}; -use crate::{IndexPage, Persistable}; +use crate::{Link, Persistable}; #[derive(Archive, Clone, Deserialize, Debug, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] pub struct UnsizedIndexPage< @@ -57,7 +57,8 @@ where T: Archive + for<'a> Serialize< Strategy, Share>, rkyv::rancor::Error>, - >, + > + Send + + Sync, ::Archived: Deserialize>, { type Utility = UnsizedIndexPageUtility; @@ -153,7 +154,11 @@ where } } - fn rebuild(&mut self) { + fn rebuild(&mut self) + where + T: Clone, + { + self.node_id = self.index_values.last().unwrap().key.clone(); self.node_id_size = self.node_id.aligned_size() as u16; self.last_value_offset = 0; let mut slots = vec![]; @@ -171,7 +176,7 @@ where T: Clone, { let index_values = self.index_values.split_off(index); - let mut new_page = UnsizedIndexPage::new_with_values(index_values); + let new_page = UnsizedIndexPage::new_with_values(index_values); self.rebuild(); new_page @@ -230,6 +235,25 @@ where file.seek(SeekFrom::Current(-(offset as i64))).await?; Self::read_value(file, len).await } + + pub fn get_node(&self) -> Vec> + where + T: Clone + Ord, + { + self.index_values + .clone() + .into_iter() + .map(|v| v.into()) + .collect() + } + + pub fn from_node(node: &[impl Into> + Clone]) -> Self + where + T: Clone + Ord + Default, + { + let values = node.iter().map(|v| v.clone().into()).collect::>(); + Self::new_with_values(values) + } } impl Persistable for UnsizedIndexPage @@ -354,6 +378,8 @@ mod test { assert_eq!(page.last_value_offset, offset as u32); assert_eq!(page.last_value_offset, page.slots.last().unwrap().0); + assert_ne!(page.node_id, split.node_id); + assert_eq!(split.slots_size, 5); let offset = split.slots.iter().map(|(_, l)| *l).sum::(); assert_eq!(split.last_value_offset, offset as u32); diff --git a/src/page/mod.rs b/src/page/mod.rs index 61a4b99..7a5b9c1 100644 --- a/src/page/mod.rs +++ b/src/page/mod.rs @@ -16,7 +16,7 @@ pub use data::DataPage; pub use header::{GeneralHeader, DATA_VERSION}; pub use index::{ get_index_page_size_from_data_length, IndexPage, IndexPageUtility, IndexValue, - TableOfContentsPage, UnsizedIndexPage, + TableOfContentsPage, UnsizedIndexPage, UnsizedIndexPageUtility, }; //pub use iterators::{DataIterator, LinksIterator}; pub use space_info::{Interval, SpaceInfoPage}; diff --git a/src/page/ty.rs b/src/page/ty.rs index 27795d5..345e07c 100644 --- a/src/page/ty.rs +++ b/src/page/ty.rs @@ -25,6 +25,8 @@ pub enum PageType { Data = 2, /// Index `Page` type. Index = 3, + /// Index for unsized type's `Page` type. + IndexUnsized = 30, /// Index's table of contests `Page` type. Is used to determine node's `PageId`. IndexTableOfContents = 31, } diff --git a/src/util/sized.rs b/src/util/sized.rs index 1477e28..6e83324 100644 --- a/src/util/sized.rs +++ b/src/util/sized.rs @@ -135,6 +135,25 @@ where } } +impl SizeMeasurable for indexset::core::pair::Pair +where + K: SizeMeasurable, + V: SizeMeasurable, +{ + fn aligned_size(&self) -> usize { + align(self.key.aligned_size() + self.value.aligned_size()) + } +} +impl SizeMeasurable for indexset::core::multipair::MultiPair +where + K: SizeMeasurable, + V: SizeMeasurable, +{ + fn aligned_size(&self) -> usize { + align(self.key.aligned_size() + self.value.aligned_size()) + } +} + /// Marks an objects that can return theirs approximate size after archiving via /// [`rkyv`]. pub trait VariableSizeMeasurable { @@ -152,6 +171,23 @@ impl VariableSizeMeasurable for String { } } +impl VariableSizeMeasurable for indexset::core::pair::Pair +where + K: VariableSizeMeasurable, +{ + fn aligned_size(length: usize) -> usize { + align(Link::default().aligned_size() + K::aligned_size(length)) + } +} +impl VariableSizeMeasurable for indexset::core::multipair::MultiPair +where + K: VariableSizeMeasurable, +{ + fn aligned_size(length: usize) -> usize { + align(Link::default().aligned_size() + K::aligned_size(length)) + } +} + #[cfg(test)] mod test { use crate::util::sized::SizeMeasurable; From 9d5e892c32cf511f75d2356eb5e5e79ac1ffde93 Mon Sep 17 00:00:00 2001 From: Handy-caT <37216852+Handy-caT@users.noreply.github.com> Date: Mon, 14 Apr 2025 14:00:40 +0300 Subject: [PATCH 7/8] clippy --- codegen/src/persistable/generator/obj_impl.rs | 8 ++++---- src/page/index/page_for_unsized.rs | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/codegen/src/persistable/generator/obj_impl.rs b/codegen/src/persistable/generator/obj_impl.rs index abef4cd..5be747c 100644 --- a/codegen/src/persistable/generator/obj_impl.rs +++ b/codegen/src/persistable/generator/obj_impl.rs @@ -141,13 +141,13 @@ impl Generator { let field_sizes = self.struct_def.fields.iter().map(|f| { let field_type_str = f.ty.to_token_stream().to_string(); if field_type_str.contains("Vec") { - self.gen_vec_size_fns(&f) + self.gen_vec_size_fns(f) } else if field_type_str.contains("String") { - self.gen_string_size_fn(&f) + self.gen_string_size_fn(f) } else if gens.contains(&field_type_str) && self.is_generic_unsized { - self.gen_generic_size_fn(&f) + self.gen_generic_size_fn(f) } else { - self.gen_primitive_size_fn(&f) + self.gen_primitive_size_fn(f) } }); diff --git a/src/page/index/page_for_unsized.rs b/src/page/index/page_for_unsized.rs index 3599df2..a9bd6ad 100644 --- a/src/page/index/page_for_unsized.rs +++ b/src/page/index/page_for_unsized.rs @@ -281,13 +281,13 @@ where let utility_bytes = utility_bytes.as_ref().to_vec(); let utility_len = utility_bytes.len(); let mut bytes = vec![0u8; data_length]; - bytes.splice(0..utility_len, utility_bytes.iter().map(|v| *v)); + bytes.splice(0..utility_len, utility_bytes.iter().copied()); for ((offset, len), value) in self.slots.iter().zip(self.index_values.iter()) { let offset = data_length - *offset as usize; let len = *len as usize; let value_bytes = rkyv::to_bytes::(value).unwrap(); - bytes.splice(offset..(offset + len), value_bytes.iter().map(|v| *v)); + bytes.splice(offset..(offset + len), value_bytes.iter().copied()); } bytes @@ -296,14 +296,14 @@ where fn from_bytes(bytes: &[u8]) -> Self { let slots_size_bytes = &bytes[0..UnsizedIndexPageUtility::::slots_size_size()]; let archived = - unsafe { rkyv::access_unchecked::<::Archived>(&slots_size_bytes) }; + unsafe { rkyv::access_unchecked::<::Archived>(slots_size_bytes) }; let slots_size = rkyv::deserialize::(archived).expect("data should be valid"); let node_id_size_bytes = &bytes[UnsizedIndexPageUtility::::slots_size_size() ..UnsizedIndexPageUtility::::node_id_size_size() + UnsizedIndexPageUtility::::node_id_size_size()]; let archived = - unsafe { rkyv::access_unchecked::<::Archived>(&node_id_size_bytes) }; + unsafe { rkyv::access_unchecked::<::Archived>(node_id_size_bytes) }; let node_id_size = rkyv::deserialize::(archived).expect("data should be valid"); let utility_len = UnsizedIndexPageUtility::::persisted_size( From ff9cd57b8f301ce2bf6e47a42d894fb4d60b41d2 Mon Sep 17 00:00:00 2001 From: Handy-caT <37216852+Handy-caT@users.noreply.github.com> Date: Tue, 15 Apr 2025 21:10:47 +0300 Subject: [PATCH 8/8] update indexset's version --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 44ca367..83ffbb4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,6 @@ derive_more = { version = "1.0.0", features = ["from", "error", "display", "into rkyv = { version = "0.8.9", features = ["uuid-1"] } lockfree = "0.5.1" uuid = { version = "1.11.0", features = ["v4"] } -# indexset = { version = "0.11.2", features = ["concurrent", "cdc", "multimap"] } -indexset = { path = "../indexset", version = "0.11.3", features = ["concurrent", "cdc", "multimap"] } +indexset = { version = "0.12.0", features = ["concurrent", "cdc", "multimap"] } +# indexset = { path = "../indexset", version = "0.11.3", features = ["concurrent", "cdc", "multimap"] } tokio = { version = "1", features = ["full"] }