Skip to content

Commit c5c91e2

Browse files
GrapeBaBaGrapeBaBa
committed
feat:add rearchive test
Signed-off-by: Chen Kai <[email protected]>
1 parent 47b7b17 commit c5c91e2

File tree

8 files changed

+183
-39
lines changed

8 files changed

+183
-39
lines changed

‎Cargo.lock

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ eth2 = { git = "https://github.com/sigp/lighthouse.git", tag = "v5.2.1" }
1616

1717
clap = "4"
1818
futures-util = "0.3"
19-
reqwest ="0.12"
19+
reqwest = "0.12"
2020
serde_json = "1.0.94"
2121
serde = "1.0"
2222
tracing = "0.1.40"
@@ -30,4 +30,5 @@ aws-sdk-s3 = "1.41.0"
3030
tokio = { version = "1.38.1", features = ["full"] }
3131
again = "0.1.2"
3232
rand = "0.8.5"
33-
once_cell = "1.19.0"
33+
once_cell = "1.19.0"
34+
hex = "0.4.3"

‎bin/archiver/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ tracing.workspace = true
2020
tokio.workspace = true
2121
eyre.workspace = true
2222
again.workspace = true
23+
hex.workspace = true
2324
blob-archiver-storage = { path = "../../crates/storage" }
2425
blob-archiver-beacon = { path = "../../crates/beacon" }
2526

‎bin/archiver/src/archiver.rs

Lines changed: 69 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,22 @@
1+
use std::sync::atomic::{AtomicU64, Ordering};
2+
use std::sync::Arc;
3+
use std::time::Duration;
4+
15
use again::RetryPolicy;
2-
use blob_archiver_beacon::beacon_client::BeaconClient;
3-
use blob_archiver_storage::{
4-
BackfillProcess, BackfillProcesses, BlobData, BlobSidecars, Header, LockFile, Storage,
5-
};
66
use eth2::types::Slot;
77
use eth2::types::{BlockHeaderData, BlockId, Hash256};
88
use eth2::Error;
99
use eyre::{eyre, Result};
1010
use serde::{Deserialize, Serialize};
11-
use std::sync::atomic::{AtomicU64, Ordering};
12-
use std::sync::Arc;
13-
use std::time::Duration;
1411
use tokio::sync::watch::Receiver;
1512
use tokio::time::{interval, sleep};
1613
use tracing::log::{debug, error, info, trace};
1714

15+
use blob_archiver_beacon::beacon_client::BeaconClient;
16+
use blob_archiver_storage::{
17+
BackfillProcess, BackfillProcesses, BlobData, BlobSidecars, Header, LockFile, Storage,
18+
};
19+
1820
#[allow(dead_code)]
1921
const LIVE_FETCH_BLOB_MAXIMUM_RETRIES: usize = 10;
2022
#[allow(dead_code)]
@@ -450,11 +452,68 @@ mod tests {
450452
use std::str::FromStr;
451453
use std::time::Duration;
452454

453-
use super::*;
454-
use blob_archiver_beacon::beacon_client::BeaconClientEth2;
455-
use blob_archiver_storage::fs::FSStorage;
456455
use eth2::{BeaconNodeHttpClient, SensitiveUrl, Timeouts};
457456

457+
use blob_archiver_beacon::beacon_client::{BeaconClientEth2, BeaconClientStub};
458+
use blob_archiver_beacon::blob_test_helper;
459+
use blob_archiver_beacon::blob_test_helper::{new_blob_sidecars, START_SLOT};
460+
use blob_archiver_storage::fs::FSStorage;
461+
462+
use super::*;
463+
464+
#[tokio::test]
465+
async fn test_rearchive_range() {
466+
let dir = &PathBuf::from("test_rearchive_range");
467+
let storage = FSStorage::new(dir.clone()).await.unwrap();
468+
tokio::fs::create_dir_all(dir).await.unwrap();
469+
let (_, rx) = tokio::sync::watch::channel(false);
470+
let beacon_client = Arc::new(BeaconClientStub::default());
471+
let archiver = Archiver::new(beacon_client.clone(), Arc::new(storage), rx);
472+
473+
let blob_sidecars_data = new_blob_sidecars(6);
474+
let blob_data = BlobData {
475+
header: Header {
476+
beacon_block_hash: *blob_test_helper::THREE,
477+
},
478+
blob_sidecars: BlobSidecars {
479+
data: blob_sidecars_data,
480+
},
481+
};
482+
let res = archiver.storage.write_blob_data(&blob_data).await;
483+
assert!(res.is_ok());
484+
485+
assert!(!archiver.storage.exists(&blob_test_helper::ONE).await);
486+
assert!(!archiver.storage.exists(&blob_test_helper::TWO).await);
487+
assert!(archiver.storage.exists(&blob_test_helper::THREE).await);
488+
assert!(!archiver.storage.exists(&blob_test_helper::FOUR).await);
489+
490+
let from = START_SLOT + 1;
491+
let to = START_SLOT + 4;
492+
493+
let result = archiver.rearchive_range(from, to).await;
494+
assert!(result.error.is_none());
495+
assert_eq!(from, result.from);
496+
assert_eq!(to, result.to);
497+
498+
assert!(archiver.storage.exists(&blob_test_helper::ONE).await);
499+
assert!(archiver.storage.exists(&blob_test_helper::TWO).await);
500+
assert!(archiver.storage.exists(&blob_test_helper::THREE).await);
501+
assert!(archiver.storage.exists(&blob_test_helper::FOUR).await);
502+
503+
assert_eq!(
504+
archiver
505+
.storage
506+
.read_blob_data(&blob_test_helper::THREE)
507+
.await
508+
.unwrap()
509+
.blob_sidecars
510+
.data
511+
.len(),
512+
4
513+
);
514+
clean_dir(dir);
515+
}
516+
458517
#[tokio::test]
459518
async fn test_persist_blobs_for_block() {
460519
let beacon_client = BeaconNodeHttpClient::new(

‎crates/beacon/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,4 @@ async-trait.workspace = true
1313
eth2.workspace = true
1414
rand.workspace = true
1515
once_cell.workspace = true
16+
hex.workspace = true

‎crates/beacon/src/beacon_client.rs

Lines changed: 61 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
1-
use std::collections::HashMap;
2-
31
use async_trait::async_trait;
42
use eth2::types::{
53
BeaconBlockHeader, BlobSidecarList, BlockHeaderAndSignature, BlockHeaderData, BlockId, EthSpec,
64
ExecutionOptimisticFinalizedResponse, GenericResponse, Hash256, MainnetEthSpec, SignatureBytes,
75
Slot,
86
};
97
use eth2::{BeaconNodeHttpClient, Error};
8+
use std::collections::HashMap;
9+
use std::str::FromStr;
1010

1111
use crate::blob_test_helper::{
1212
new_blob_sidecars, FIVE, FOUR, ONE, ORIGIN_BLOCK, START_SLOT, THREE, TWO,
@@ -76,7 +76,7 @@ impl Default for BeaconClientStub<MainnetEthSpec> {
7676
}
7777
};
7878

79-
let start_slot: Slot = START_SLOT;
79+
let start_slot = Slot::new(START_SLOT);
8080
let origin_blobs = new_blob_sidecars(1);
8181
let one_blobs = new_blob_sidecars(2);
8282
let two_blobs = new_blob_sidecars(0);
@@ -87,17 +87,36 @@ impl Default for BeaconClientStub<MainnetEthSpec> {
8787
BeaconClientStub {
8888
headers: HashMap::from([
8989
(
90-
ORIGIN_BLOCK.to_string(),
91-
make_header(start_slot, *ORIGIN_BLOCK, Hash256::from_slice(&[9, 9, 9])),
90+
format!("0x{}", hex::encode(ORIGIN_BLOCK.as_bytes())),
91+
make_header(
92+
start_slot,
93+
*ORIGIN_BLOCK,
94+
Hash256::from_str(
95+
"0x0909090000000000000000000000000000000000000000000000000000000000",
96+
)
97+
.unwrap(),
98+
),
9299
),
93100
(
94-
ONE.to_string(),
101+
format!("0x{}", hex::encode(ONE.as_bytes())),
95102
make_header(start_slot + 1, *ONE, *ORIGIN_BLOCK),
96103
),
97-
(TWO.to_string(), make_header(start_slot + 2, *TWO, *ONE)),
98-
(THREE.to_string(), make_header(start_slot + 3, *THREE, *TWO)),
99-
(FOUR.to_string(), make_header(start_slot + 4, *FOUR, *THREE)),
100-
(FIVE.to_string(), make_header(start_slot + 5, *FIVE, *FOUR)),
104+
(
105+
format!("0x{}", hex::encode(TWO.as_bytes())),
106+
make_header(start_slot + 2, *TWO, *ONE),
107+
),
108+
(
109+
format!("0x{}", hex::encode(THREE.as_bytes())),
110+
make_header(start_slot + 3, *THREE, *TWO),
111+
),
112+
(
113+
format!("0x{}", hex::encode(FOUR.as_bytes())),
114+
make_header(start_slot + 4, *FOUR, *THREE),
115+
),
116+
(
117+
format!("0x{}", hex::encode(FIVE.as_bytes())),
118+
make_header(start_slot + 5, *FIVE, *FOUR),
119+
),
101120
(
102121
"head".to_string(),
103122
make_header(start_slot + 5, *FIVE, *FOUR),
@@ -108,7 +127,14 @@ impl Default for BeaconClientStub<MainnetEthSpec> {
108127
),
109128
(
110129
start_slot.as_u64().to_string(),
111-
make_header(start_slot, *ORIGIN_BLOCK, Hash256::from_slice(&[9, 9, 9])),
130+
make_header(
131+
start_slot,
132+
*ORIGIN_BLOCK,
133+
Hash256::from_str(
134+
"0x0909090000000000000000000000000000000000000000000000000000000000",
135+
)
136+
.unwrap(),
137+
),
112138
),
113139
(
114140
(start_slot + 1).as_u64().to_string(),
@@ -133,12 +159,30 @@ impl Default for BeaconClientStub<MainnetEthSpec> {
133159
]),
134160

135161
blobs: HashMap::from([
136-
(ORIGIN_BLOCK.to_string(), origin_blobs.clone()),
137-
(ONE.to_string(), one_blobs.clone()),
138-
(TWO.to_string(), two_blobs.clone()),
139-
(THREE.to_string(), three_blobs.clone()),
140-
(FOUR.to_string(), four_blobs.clone()),
141-
(FIVE.to_string(), five_blobs.clone()),
162+
(
163+
format!("0x{}", hex::encode(ORIGIN_BLOCK.as_bytes())),
164+
origin_blobs.clone(),
165+
),
166+
(
167+
format!("0x{}", hex::encode(ONE.as_bytes())),
168+
one_blobs.clone(),
169+
),
170+
(
171+
format!("0x{}", hex::encode(TWO.as_bytes())),
172+
two_blobs.clone(),
173+
),
174+
(
175+
format!("0x{}", hex::encode(THREE.as_bytes())),
176+
three_blobs.clone(),
177+
),
178+
(
179+
format!("0x{}", hex::encode(FOUR.as_bytes())),
180+
four_blobs.clone(),
181+
),
182+
(
183+
format!("0x{}", hex::encode(FIVE.as_bytes())),
184+
five_blobs.clone(),
185+
),
142186
("head".to_string(), five_blobs.clone()),
143187
("finalized".to_string(), three_blobs.clone()),
144188
(start_slot.as_u64().to_string(), origin_blobs.clone()),

‎crates/beacon/src/blob_test_helper.rs

Lines changed: 45 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,56 @@ use std::sync::Arc;
33
use eth2::types::test_utils::TestRandom;
44
use eth2::types::{
55
Blob, BlobSidecar, BlobSidecarList, FixedVector, Hash256, KzgCommitment, KzgProof,
6-
MainnetEthSpec, SignedBeaconBlockHeader, Slot,
6+
MainnetEthSpec, SignedBeaconBlockHeader,
77
};
88
use once_cell::sync::Lazy;
99

10-
pub static ORIGIN_BLOCK: Lazy<Hash256> = Lazy::new(|| Hash256::from_slice(&[9, 9, 9, 9, 9]));
11-
pub static ONE: Lazy<Hash256> = Lazy::new(|| Hash256::from_slice(&[1]));
12-
pub static TWO: Lazy<Hash256> = Lazy::new(|| Hash256::from_slice(&[2]));
13-
pub static THREE: Lazy<Hash256> = Lazy::new(|| Hash256::from_slice(&[3]));
14-
pub static FOUR: Lazy<Hash256> = Lazy::new(|| Hash256::from_slice(&[4]));
15-
pub static FIVE: Lazy<Hash256> = Lazy::new(|| Hash256::from_slice(&[5]));
10+
pub static ORIGIN_BLOCK: Lazy<Hash256> = Lazy::new(|| {
11+
Hash256::from([
12+
0x09, 0x09, 0x09, 0x09, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
13+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
14+
0x00, 0x00,
15+
])
16+
});
17+
pub static ONE: Lazy<Hash256> = Lazy::new(|| {
18+
Hash256::from([
19+
0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
20+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
21+
0x00, 0x00,
22+
])
23+
});
24+
pub static TWO: Lazy<Hash256> = Lazy::new(|| {
25+
Hash256::from([
26+
0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
27+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
28+
0x00, 0x00,
29+
])
30+
});
31+
pub static THREE: Lazy<Hash256> = Lazy::new(|| {
32+
Hash256::from([
33+
0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
34+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
35+
0x00, 0x00,
36+
])
37+
});
38+
pub static FOUR: Lazy<Hash256> = Lazy::new(|| {
39+
Hash256::from([
40+
0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
41+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
42+
0x00, 0x00,
43+
])
44+
});
45+
pub static FIVE: Lazy<Hash256> = Lazy::new(|| {
46+
Hash256::from([
47+
0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
48+
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
49+
0x00, 0x00,
50+
])
51+
});
1652

17-
pub const START_SLOT: Slot = Slot::new(10);
53+
pub const START_SLOT: u64 = 10;
1854
#[allow(dead_code)]
19-
pub const END_SLOT: Slot = Slot::new(15);
55+
pub const END_SLOT: u64 = 15;
2056

2157
pub fn new_blob_sidecar(i: u64) -> BlobSidecar<MainnetEthSpec> {
2258
let mut rng = rand::thread_rng();

‎crates/beacon/src/lib.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
pub mod beacon_client;
2-
mod blob_test_helper;
2+
pub mod blob_test_helper;

0 commit comments

Comments
 (0)