@@ -147,101 +147,66 @@ impl Options {
147
147
}
148
148
149
149
pub fn from_env ( ) -> Result < Self > {
150
- let mut options = Self :: default ( ) ;
151
- if let Ok ( key) = std:: env:: var ( "LIBSQL_BOTTOMLESS_ENDPOINT" ) {
152
- options. aws_endpoint = Some ( key) ;
153
- }
154
- if let Ok ( bucket_name) = std:: env:: var ( "LIBSQL_BOTTOMLESS_BUCKET" ) {
155
- options. bucket_name = bucket_name;
156
- }
157
- if let Ok ( seconds) = std:: env:: var ( "LIBSQL_BOTTOMLESS_BATCH_INTERVAL_SECS" ) {
158
- if let Ok ( seconds) = seconds. parse :: < u64 > ( ) {
159
- options. max_batch_interval = Duration :: from_secs ( seconds) ;
160
- }
161
- }
162
- if let Ok ( access_key_id) = std:: env:: var ( "LIBSQL_BOTTOMLESS_AWS_ACCESS_KEY_ID" ) {
163
- options. access_key_id = Some ( access_key_id) ;
164
- }
165
- if let Ok ( secret_access_key) = std:: env:: var ( "LIBSQL_BOTTOMLESS_AWS_SECRET_ACCESS_KEY" ) {
166
- options. secret_access_key = Some ( secret_access_key) ;
167
- }
168
- if let Ok ( region) = std:: env:: var ( "LIBSQL_BOTTOMLESS_AWS_DEFAULT_REGION" ) {
169
- options. region = Some ( region) ;
170
- }
171
- if let Ok ( count) = std:: env:: var ( "LIBSQL_BOTTOMLESS_BATCH_MAX_FRAMES" ) {
172
- match count. parse :: < usize > ( ) {
173
- Ok ( count) => options. max_frames_per_batch = count,
174
- Err ( e) => {
175
- bail ! (
176
- "Invalid LIBSQL_BOTTOMLESS_BATCH_MAX_FRAMES environment variable: {}" ,
177
- e
178
- )
179
- }
150
+ fn env_var ( key : & str ) -> Result < String > {
151
+ match std:: env:: var ( key) {
152
+ Ok ( res) => Ok ( res) ,
153
+ Err ( _) => bail ! ( "{} environment variable not set" , key) ,
180
154
}
181
155
}
182
- if let Ok ( parallelism) = std:: env:: var ( "LIBSQL_BOTTOMLESS_S3_PARALLEL_MAX" ) {
183
- match parallelism. parse :: < usize > ( ) {
184
- Ok ( parallelism) => options. s3_upload_max_parallelism = parallelism,
185
- Err ( e) => bail ! (
186
- "Invalid LIBSQL_BOTTOMLESS_S3_PARALLEL_MAX environment variable: {}" ,
187
- e
188
- ) ,
156
+ fn env_var_or < S : ToString > ( key : & str , default_value : S ) -> String {
157
+ match std:: env:: var ( key) {
158
+ Ok ( res) => res,
159
+ Err ( _) => default_value. to_string ( ) ,
189
160
}
190
161
}
191
- if let Ok ( swap_after) = std:: env:: var ( "LIBSQL_BOTTOMLESS_RESTORE_TXN_SWAP_THRESHOLD" ) {
192
- match swap_after. parse :: < u32 > ( ) {
193
- Ok ( swap_after) => options. restore_transaction_page_swap_after = swap_after,
194
- Err ( e) => bail ! (
195
- "Invalid LIBSQL_BOTTOMLESS_RESTORE_TXN_SWAP_THRESHOLD environment variable: {}" ,
196
- e
197
- ) ,
198
- }
199
- }
200
- if let Ok ( fpath) = std:: env:: var ( "LIBSQL_BOTTOMLESS_RESTORE_TXN_FILE" ) {
201
- options. restore_transaction_cache_fpath = fpath;
202
- }
203
- if let Ok ( compression) = std:: env:: var ( "LIBSQL_BOTTOMLESS_COMPRESSION" ) {
204
- match CompressionKind :: parse ( & compression) {
205
- Ok ( compression) => options. use_compression = compression,
206
- Err ( e) => bail ! (
207
- "Invalid LIBSQL_BOTTOMLESS_COMPRESSION environment variable: {}" ,
208
- e
209
- ) ,
210
- }
211
- }
212
- if let Ok ( verify) = std:: env:: var ( "LIBSQL_BOTTOMLESS_VERIFY_CRC" ) {
213
- match verify. to_lowercase ( ) . as_ref ( ) {
214
- "yes" | "true" | "1" | "y" | "t" => options. verify_crc = true ,
215
- "no" | "false" | "0" | "n" | "f" => options. verify_crc = false ,
216
- other => bail ! (
217
- "Invalid LIBSQL_BOTTOMLESS_VERIFY_CRC environment variable: {}" ,
218
- other
219
- ) ,
220
- }
221
- }
222
- Ok ( options)
223
- }
224
- }
225
162
226
- impl Default for Options {
227
- fn default ( ) -> Self {
228
- let db_id = std:: env:: var ( "LIBSQL_BOTTOMLESS_DATABASE_ID" ) . ok ( ) ;
229
- Options {
230
- create_bucket_if_not_exists : true ,
231
- verify_crc : true ,
232
- use_compression : CompressionKind :: Gzip ,
233
- max_batch_interval : Duration :: from_secs ( 15 ) ,
234
- max_frames_per_batch : 500 , // basically half of the default SQLite checkpoint size
235
- s3_upload_max_parallelism : 32 ,
236
- restore_transaction_page_swap_after : 1000 ,
163
+ let db_id = env_var ( "LIBSQL_BOTTOMLESS_DATABASE_ID" ) . ok ( ) ;
164
+ let aws_endpoint = env_var ( "LIBSQL_BOTTOMLESS_ENDPOINT" ) . ok ( ) ;
165
+ let bucket_name = env_var_or ( "LIBSQL_BOTTOMLESS_BUCKET" , "bottomless" ) ;
166
+ let max_batch_interval = Duration :: from_secs (
167
+ env_var_or ( "LIBSQL_BOTTOMLESS_BATCH_INTERVAL_SECS" , 15 ) . parse :: < u64 > ( ) ?,
168
+ ) ;
169
+ let access_key_id = env_var ( "LIBSQL_BOTTOMLESS_AWS_ACCESS_KEY_ID" ) . ok ( ) ;
170
+ let secret_access_key = env_var ( "LIBSQL_BOTTOMLESS_AWS_SECRET_ACCESS_KEY" ) . ok ( ) ;
171
+ let region = env_var ( "LIBSQL_BOTTOMLESS_AWS_DEFAULT_REGION" ) . ok ( ) ;
172
+ let max_frames_per_batch =
173
+ env_var_or ( "LIBSQL_BOTTOMLESS_BATCH_MAX_FRAMES" , 500 ) . parse :: < usize > ( ) ?;
174
+ let s3_upload_max_parallelism =
175
+ env_var_or ( "LIBSQL_BOTTOMLESS_S3_PARALLEL_MAX" , 32 ) . parse :: < usize > ( ) ?;
176
+ let restore_transaction_page_swap_after =
177
+ env_var_or ( "LIBSQL_BOTTOMLESS_RESTORE_TXN_SWAP_THRESHOLD" , 1000 ) . parse :: < u32 > ( ) ?;
178
+ let restore_transaction_cache_fpath =
179
+ env_var_or ( "LIBSQL_BOTTOMLESS_RESTORE_TXN_FILE" , ".bottomless.restore" ) ;
180
+ let use_compression =
181
+ CompressionKind :: parse ( & env_var_or ( "LIBSQL_BOTTOMLESS_COMPRESSION" , "gz" ) )
182
+ . map_err ( |e| anyhow ! ( "unknown compression kind: {}" , e) ) ?;
183
+ let verify_crc = match env_var_or ( "LIBSQL_BOTTOMLESS_VERIFY_CRC" , true )
184
+ . to_lowercase ( )
185
+ . as_ref ( )
186
+ {
187
+ "yes" | "true" | "1" | "y" | "t" => true ,
188
+ "no" | "false" | "0" | "n" | "f" => false ,
189
+ other => bail ! (
190
+ "Invalid LIBSQL_BOTTOMLESS_VERIFY_CRC environment variable: {}" ,
191
+ other
192
+ ) ,
193
+ } ;
194
+ Ok ( Options {
237
195
db_id,
238
- aws_endpoint : None ,
239
- access_key_id : None ,
240
- secret_access_key : None ,
241
- region : None ,
242
- restore_transaction_cache_fpath : ".bottomless.restore" . to_string ( ) ,
243
- bucket_name : "bottomless" . to_string ( ) ,
244
- }
196
+ create_bucket_if_not_exists : true ,
197
+ verify_crc,
198
+ use_compression,
199
+ max_batch_interval,
200
+ max_frames_per_batch,
201
+ s3_upload_max_parallelism,
202
+ restore_transaction_page_swap_after,
203
+ aws_endpoint,
204
+ access_key_id,
205
+ secret_access_key,
206
+ region,
207
+ restore_transaction_cache_fpath,
208
+ bucket_name,
209
+ } )
245
210
}
246
211
}
247
212
@@ -276,13 +241,10 @@ impl Replicator {
276
241
}
277
242
278
243
let db_path = db_path. into ( ) ;
279
- let db_name = {
280
- let db_id = options. db_id . unwrap_or_default ( ) ;
281
- let name = match db_path. find ( '/' ) {
282
- Some ( index) => & db_path[ ..index] ,
283
- None => & db_path,
284
- } ;
285
- db_id + ":" + name
244
+ let db_name = if let Some ( db_id) = options. db_id . clone ( ) {
245
+ db_id
246
+ } else {
247
+ bail ! ( "database id was not set" )
286
248
} ;
287
249
tracing:: debug!( "Database path: '{}', name: '{}'" , db_path, db_name) ;
288
250
@@ -963,7 +925,7 @@ impl Replicator {
963
925
async fn restore_from (
964
926
& mut self ,
965
927
generation : Uuid ,
966
- utc_time : Option < NaiveDateTime > ,
928
+ timestamp : Option < NaiveDateTime > ,
967
929
) -> Result < ( RestoreAction , bool ) > {
968
930
if let Some ( tombstone) = self . get_tombstone ( ) . await ? {
969
931
if let Some ( timestamp) = Self :: generation_to_timestamp ( & generation) {
@@ -991,11 +953,31 @@ impl Replicator {
991
953
992
954
// at this point we know, we should do a full restore
993
955
994
- tokio:: fs:: rename ( & self . db_path , format ! ( "{}.bottomless.backup" , self . db_path) )
995
- . await
996
- . ok ( ) ; // Best effort
956
+ let backup_path = format ! ( "{}.bottomless.backup" , self . db_path) ;
957
+ tokio:: fs:: rename ( & self . db_path , & backup_path) . await . ok ( ) ; // Best effort
958
+ match self . full_restore ( generation, timestamp, last_frame) . await {
959
+ Ok ( result) => {
960
+ let elapsed = Instant :: now ( ) - start_ts;
961
+ tracing:: info!( "Finished database restoration in {:?}" , elapsed) ;
962
+ tokio:: fs:: remove_file ( backup_path) . await . ok ( ) ;
963
+ Ok ( result)
964
+ }
965
+ Err ( e) => {
966
+ tracing:: error!( "failed to restore the database: {}. Rollback" , e) ;
967
+ tokio:: fs:: rename ( & backup_path, & self . db_path ) . await . ok ( ) ;
968
+ Err ( e)
969
+ }
970
+ }
971
+ }
972
+
973
+ async fn full_restore (
974
+ & mut self ,
975
+ generation : Uuid ,
976
+ timestamp : Option < NaiveDateTime > ,
977
+ last_frame : u32 ,
978
+ ) -> Result < ( RestoreAction , bool ) > {
997
979
let _ = self . remove_wal_files ( ) . await ; // best effort, WAL files may not exists
998
- let mut db = tokio :: fs :: OpenOptions :: new ( )
980
+ let mut db = OpenOptions :: new ( )
999
981
. create ( true )
1000
982
. read ( true )
1001
983
. write ( true )
@@ -1052,7 +1034,7 @@ impl Replicator {
1052
1034
page_size as usize ,
1053
1035
last_frame,
1054
1036
checksum,
1055
- utc_time ,
1037
+ timestamp ,
1056
1038
& mut db,
1057
1039
)
1058
1040
. await ?;
@@ -1063,8 +1045,6 @@ impl Replicator {
1063
1045
}
1064
1046
1065
1047
db. shutdown ( ) . await ?;
1066
- let elapsed = Instant :: now ( ) - start_ts;
1067
- tracing:: info!( "Finished database restoration in {:?}" , elapsed) ;
1068
1048
1069
1049
if applied_wal_frame {
1070
1050
tracing:: info!( "WAL file has been applied onto database file in generation {}. Requesting snapshot." , generation) ;
0 commit comments