Skip to content

Unsized index page #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Handy-caT merged 8 commits into from
Apr 15, 2025
Merged

Unsized index page #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4315428
start page for unsized
Handy-caT Apr 4, 2025
894fae9
WIP
Handy-caT Apr 5, 2025
819ea08
fix unsized page logic
Handy-caT Apr 7, 2025
31c3842
add split logic for the page
Handy-caT Apr 8, 2025
74db64c
correction
Handy-caT Apr 8, 2025
81dd05b
add VariableSizeMeasure and corrections
Handy-caT Apr 14, 2025
9d5e892
clippy
Handy-caT Apr 14, 2025
ff9cd57
update indexset's version
Handy-caT Apr 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,6 @@ derive_more = { version = "1.0.0", features = ["from", "error", "display", "into
rkyv = { version = "0.8.9", features = ["uuid-1"] }
lockfree = "0.5.1"
uuid = { version = "1.11.0", features = ["v4"] }
indexset = { version = "0.11.2", features = ["concurrent", "cdc", "multimap"] }
indexset = { version = "0.12.0", features = ["concurrent", "cdc", "multimap"] }
# indexset = { path = "../indexset", version = "0.11.3", features = ["concurrent", "cdc", "multimap"] }
tokio = { version = "1", features = ["full"] }
8 changes: 8 additions & 0 deletions codegen/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
mod persistable;
mod size_measure;
mod variable_size_measure;

use proc_macro::TokenStream;

Expand All @@ -10,6 +11,13 @@ pub fn size_measure(input: TokenStream) -> TokenStream {
.into()
}

#[proc_macro_derive(VariableSizeMeasure)]
pub fn variable_size_measure(input: TokenStream) -> TokenStream {
variable_size_measure::expand(&(input.into()))
.unwrap_or_else(|e| e.to_compile_error())
.into()
}

#[proc_macro_derive(Persistable, attributes(persistable))]
pub fn persistable(input: TokenStream) -> TokenStream {
persistable::expand(&(input.into()))
Expand Down
2 changes: 2 additions & 0 deletions codegen/src/persistable/generator/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,13 @@ use syn::ItemStruct;

pub struct Generator {
pub is_full_row: bool,
pub is_generic_unsized: bool,
pub struct_def: ItemStruct,
}

pub struct PersistableAttributes {
pub is_full_row: bool,
pub is_generic_unsized: bool,
}

impl Generator {
Expand Down
238 changes: 151 additions & 87 deletions codegen/src/persistable/generator/obj_impl.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use crate::persistable::generator::persistable_impl::is_primitive;
use crate::persistable::generator::Generator;

use quote::{quote, ToTokens};
use syn::GenericParam;
use syn::{Field, GenericParam};

impl Generator {
pub fn gen_obj_impl_def(&self) -> TokenStream {
Expand Down Expand Up @@ -55,120 +55,184 @@ impl Generator {
}

pub fn gen_full_obj_size_fn(&self) -> TokenStream {
let contains_vec = self
let size_fields: Vec<_> = self
.struct_def
.fields
.iter()
.any(|f| f.ty.to_token_stream().to_string().contains("Vec"));
if contains_vec {
let sizes: Vec<_> = self
.struct_def
.fields
.iter()
.map(|f| {
let fn_ident = Ident::new(
format!("{}_size", f.ident.clone().unwrap()).as_str(),
Span::call_site(),
);

if f.ty.to_token_stream().to_string().contains("Vec") {
quote! {
Self::#fn_ident(length)
}
} else {
quote! {
Self::#fn_ident()
}
}
})
.collect();
quote! {
pub fn persisted_size(length: usize) -> usize {
#(#sizes)+*
.enumerate()
.filter(|(_, f)| f.ident.clone().unwrap().to_string().contains("size"))
.map(|(_, f)| {
let ident = f.ident.as_ref().unwrap();
quote! {
#ident: usize
}
}
} else {
let sizes: Vec<_> = self
.struct_def
.fields
.iter()
.map(|f| {
let fn_ident = Ident::new(
format!("{}_size", f.ident.clone().unwrap()).as_str(),
Span::call_site(),
);

quote! {
Self::#fn_ident()
}
})
.collect();
quote! {
pub fn persisted_size() -> usize {
#(#sizes)+*
})
.collect();
let gens: Vec<_> = self
.struct_def
.generics
.params
.iter()
.filter_map(|p| {
if let GenericParam::Type(t) = p {
Some(t.ident.to_string())
} else {
None
}
}
}
}
})
.collect();

fn gen_field_sizes_fns(&self) -> TokenStream {
let field_sizes = self.struct_def.fields.iter().map(|f| {
if f.ty.to_token_stream().to_string().contains("Vec") {
let ty = &f.ty;
let inner_ty_str = ty.to_token_stream().to_string().replace(" ", "");
let mut inner_ty_str = inner_ty_str.replace("Vec<", "");
inner_ty_str.pop();
let inner_ty: TokenStream = inner_ty_str.parse().unwrap();
let value_fn_ident = Ident::new(
format!("{}_value_size", f.ident.clone().unwrap()).as_str(),
Span::call_site(),
);
let sizes: Vec<_> = self
.struct_def
.fields
.iter()
.map(|f| {
let fn_ident = Ident::new(
format!("{}_size", f.ident.clone().unwrap()).as_str(),
Span::call_site(),
);
let len_in_vec = if is_primitive(&inner_ty_str) {
let size_ident = if size_fields.len() == 1 {
quote! {
align(length * <#inner_ty as Default>::default().aligned_size()) + 8
size
}
} else {
quote! {
length * align8(<#inner_ty as Default>::default().aligned_size()) + 8
#fn_ident
}
};
let len_value = if is_primitive(&inner_ty_str) {

let field_type_str = f.ty.to_token_stream().to_string();
if field_type_str.contains("Vec") || field_type_str.contains("String") {
quote! {
<#inner_ty as Default>::default().aligned_size()
Self::#fn_ident(#size_ident)
}
} else {
} else if gens.contains(&field_type_str) && self.is_generic_unsized {
quote! {
align8(<#inner_ty as Default>::default().aligned_size())
}
};
quote! {
pub fn #value_fn_ident() -> usize {
#len_value
#size_ident
}
pub fn #fn_ident(length: usize) -> usize {
#len_in_vec
} else {
quote! {
Self::#fn_ident()
}
}
} else {
let ident = &f.ty;
let fn_ident = Ident::new(
format!("{}_size", f.ident.clone().unwrap()).as_str(),
Span::call_site(),
);
quote! {
pub fn #fn_ident() -> usize {
<#ident as Default>::default().aligned_size()
}
})
.collect();
quote! {
pub fn persisted_size(#(#size_fields),*) -> usize {
#(#sizes)+*
}
}
}

fn gen_field_sizes_fns(&self) -> TokenStream {
let gens: Vec<_> = self
.struct_def
.generics
.params
.iter()
.filter_map(|p| {
if let GenericParam::Type(t) = p {
Some(t.ident.to_string())
} else {
None
}
})
.collect();
let field_sizes = self.struct_def.fields.iter().map(|f| {
let field_type_str = f.ty.to_token_stream().to_string();
if field_type_str.contains("Vec") {
self.gen_vec_size_fns(f)
} else if field_type_str.contains("String") {
self.gen_string_size_fn(f)
} else if gens.contains(&field_type_str) && self.is_generic_unsized {
self.gen_generic_size_fn(f)
} else {
self.gen_primitive_size_fn(f)
}
});

quote! {
#(#field_sizes)*
}
}

fn gen_primitive_size_fn(&self, f: &Field) -> TokenStream {
let ty = &f.ty;
let fn_ident = Ident::new(
format!("{}_size", f.ident.clone().unwrap()).as_str(),
Span::call_site(),
);
quote! {
pub fn #fn_ident() -> usize {
<#ty as Default>::default().aligned_size()
}
}
}

fn gen_string_size_fn(&self, f: &Field) -> TokenStream {
let fn_ident = Ident::new(
format!("{}_size", f.ident.clone().unwrap()).as_str(),
Span::call_site(),
);
quote! {
pub fn #fn_ident(length: usize) -> usize {
align(length + 8)
}
}
}

fn gen_generic_size_fn(&self, f: &Field) -> TokenStream {
let fn_ident = Ident::new(
format!("{}_size", f.ident.clone().unwrap()).as_str(),
Span::call_site(),
);
let ty_ = &f.ty;
quote! {
pub fn #fn_ident(length: usize) -> usize {
<#ty_ as VariableSizeMeasurable>::aligned_size(length)
}
}
}

fn gen_vec_size_fns(&self, f: &Field) -> TokenStream {
let ty = &f.ty;
let inner_ty_str = ty.to_token_stream().to_string().replace(" ", "");
let mut inner_ty_str = inner_ty_str.replace("Vec<", "");
inner_ty_str.pop();
let inner_ty: TokenStream = inner_ty_str.parse().unwrap();
let value_fn_ident = Ident::new(
format!("{}_value_size", f.ident.clone().unwrap()).as_str(),
Span::call_site(),
);
let fn_ident = Ident::new(
format!("{}_size", f.ident.clone().unwrap()).as_str(),
Span::call_site(),
);
let len_in_vec = if is_primitive(&inner_ty_str) {
quote! {
align(length * <#inner_ty as Default>::default().aligned_size()) + 8
}
} else {
quote! {
length * align8(<#inner_ty as Default>::default().aligned_size()) + 8
}
};
let len_value = if is_primitive(&inner_ty_str) {
quote! {
<#inner_ty as Default>::default().aligned_size()
}
} else {
quote! {
align8(<#inner_ty as Default>::default().aligned_size())
}
};
quote! {
pub fn #value_fn_ident() -> usize {
#len_value
}
pub fn #fn_ident(length: usize) -> usize {
#len_in_vec
}
}
}
}
Loading
Loading