apollo_infra_utils,apollo_integration_tests: fix max ports gen, support hybrid node args (#9671)

Author: Itay-Tsabary-Starkware

.github/workflows/hybrid_system_test.yaml

@@ -297,7 +297,7 @@ jobs:
         run: chmod +x ./target/debug/sequencer_node_setup ./target/debug/sequencer_simulator
 
       - name: Create storage files
-        run: ./target/debug/sequencer_node_setup --output-base-dir ./output --data-prefix-path /data --n-consolidated 1 --n-distributed 0
+        run: ./target/debug/sequencer_node_setup --output-base-dir ./output --data-prefix-path /data --n-consolidated 1 --n-hybrid 0 --n-distributed 0
 
       - name: Export application config dir
         run: |

Cargo.lock

@@ -7,7 +7,7 @@ license-file.workspace = true
 description = "Infrastructure utility."
 
 [features]
-testing = ["colored", "dep:assert-json-diff", "socket2", "tempfile"]
+testing = ["colored", "dep:assert-json-diff", "socket2", "strum", "tempfile"]
 
 [lints]
 workspace = true
@@ -20,6 +20,7 @@ num_enum.workspace = true
 serde = { workspace = true, features = ["derive"] }
 serde_json.workspace = true
 socket2 = { workspace = true, optional = true }
+strum = { workspace = true, optional = true }
 tempfile = { workspace = true, optional = true }
 thiserror.workspace = true
 tokio = { workspace = true, features = ["process", "rt", "time"] }
@@ -32,6 +33,7 @@ nix.workspace = true
 pretty_assertions.workspace = true
 rstest.workspace = true
 socket2.workspace = true
+strum.workspace = true
 tempfile.workspace = true
 tokio = { workspace = true, features = ["macros", "rt", "signal", "sync"] }
 toml.workspace = true

crates/apollo_infra_utils/Cargo.toml

@@ -4,11 +4,13 @@ use assert_json_diff::{assert_json_matches_no_panic, CompareMode, Config};
 use num_enum::IntoPrimitive;
 use serde::Serialize;
 use socket2::{Domain, Socket, Type};
+use strum::EnumCount;
 use tracing::{info, instrument};
 
 const PORTS_PER_INSTANCE: u16 = 60;
 pub const MAX_NUMBER_OF_INSTANCES_PER_TEST: u16 = 28;
-const MAX_NUMBER_OF_TESTS: u16 = 11;
+#[allow(clippy::as_conversions)]
+const MAX_NUMBER_OF_TESTS: u16 = TestIdentifier::COUNT as u16;
 const BASE_PORT: u16 = 11000;
 
 // Ensure available ports don't exceed u16::MAX.
@@ -21,7 +23,7 @@ const _: () = {
 };
 
 #[repr(u16)]
-#[derive(Debug, Copy, Clone, IntoPrimitive)]
+#[derive(Debug, Copy, Clone, IntoPrimitive, EnumCount)]
 // TODO(Nadin): Come up with a better name for this enum.
 pub enum TestIdentifier {
     EndToEndFlowTest,

crates/apollo_infra_utils/src/test_utils.rs

@@ -18,12 +18,14 @@ async fn main() {
     const N_CONSOLIDATED_SEQUENCERS: usize = 2;
     /// The number of distributed remote sequencers that participate in the test.
     const N_DISTRIBUTED_SEQUENCERS: usize = 0;
-
+    /// The number of hybrid sequencers that participate in the test.
+    const N_HYBRID_SEQUENCERS: usize = 0;
     const CENTRAL_SYNC_NODE: usize = 1;
 
     let mut integration_test_manager = IntegrationTestManager::new(
         N_CONSOLIDATED_SEQUENCERS,
         N_DISTRIBUTED_SEQUENCERS,
+        N_HYBRID_SEQUENCERS,
         None,
         TestIdentifier::SyncFlowIntegrationTest,
     )

crates/apollo_integration_tests/src/bin/sequencer_node_end_to_end_integration_tests/integration_test_central_and_p2p_sync_flow.rs

@@ -13,12 +13,15 @@ async fn main() {
     /// The number of consolidated local sequencers that participate in the test.
     const N_CONSOLIDATED_SEQUENCERS: usize = 3;
     /// The number of distributed remote sequencers that participate in the test.
-    const N_DISTRIBUTED_SEQUENCERS: usize = 2;
+    const N_DISTRIBUTED_SEQUENCERS: usize = 1;
+    /// The number of hybrid sequencers that participate in the test.
+    const N_HYBRID_SEQUENCERS: usize = 1;
 
     // Get the sequencer configurations.
     let mut integration_test_manager = IntegrationTestManager::new(
         N_CONSOLIDATED_SEQUENCERS,
         N_DISTRIBUTED_SEQUENCERS,
+        N_HYBRID_SEQUENCERS,
         None,
         TestIdentifier::PositiveFlowIntegrationTest,
     )

crates/apollo_integration_tests/src/bin/sequencer_node_end_to_end_integration_tests/integration_test_positive_flow.rs

@@ -25,7 +25,9 @@ async fn main() {
     /// The number of consolidated local sequencers that participate in the test.
     const N_CONSOLIDATED_SEQUENCERS: usize = 1;
     /// The number of distributed remote sequencers that participate in the test.
-    const N_DISTRIBUTED_SEQUENCERS: usize = 2;
+    const N_DISTRIBUTED_SEQUENCERS: usize = 1;
+    /// The number of hybrid sequencers that participate in the test.
+    const N_HYBRID_SEQUENCERS: usize = 1;
     // The indices of the nodes that we will be shutting down.
     // The test restarts a hybrid node and shuts down a non-consolidated (hybrid/distributed) node.
     const RESTART_NODE: usize = N_CONSOLIDATED_SEQUENCERS;
@@ -35,6 +37,7 @@ async fn main() {
     let mut integration_test_manager = IntegrationTestManager::new(
         N_CONSOLIDATED_SEQUENCERS,
         N_DISTRIBUTED_SEQUENCERS,
+        N_HYBRID_SEQUENCERS,
         None,
         TestIdentifier::RestartFlowIntegrationTest,
     )

crates/apollo_integration_tests/src/bin/sequencer_node_end_to_end_integration_tests/integration_test_restart_flow.rs

@@ -25,6 +25,8 @@ async fn main() {
     const N_L1_HANDLER_TXS: usize = 0;
     /// The number of consolidated local sequencers that participate in the test.
     const N_CONSOLIDATED_SEQUENCERS: usize = 5;
+    /// The number of hybrid sequencers that participate in the test.
+    const N_HYBRID_SEQUENCERS: usize = 0;
     /// The number of distributed remote sequencers that participate in the test.
     const N_DISTRIBUTED_SEQUENCERS: usize = 0;
 
@@ -36,6 +38,7 @@ async fn main() {
     let mut integration_test_manager = IntegrationTestManager::new(
         N_CONSOLIDATED_SEQUENCERS,
         N_DISTRIBUTED_SEQUENCERS,
+        N_HYBRID_SEQUENCERS,
         None,
         TestIdentifier::RevertFlowIntegrationTest,
     )

crates/apollo_integration_tests/src/bin/sequencer_node_end_to_end_integration_tests/integration_test_revert_flow.rs

@@ -27,6 +27,7 @@ async fn main() {
     let test_manager = IntegrationTestManager::new(
         args.n_consolidated,
         args.n_distributed,
+        args.n_hybrid,
         Some(custom_paths),
         // TODO(Tsabary/Nadin): add a different identifier.
         TestIdentifier::PositiveFlowIntegrationTest,
@@ -58,6 +59,9 @@ struct Args {
     #[arg(long)]
     n_distributed: usize,
 
+    #[arg(long)]
+    n_hybrid: usize,
+
     #[arg(long)]
     output_base_dir: String,
 

crates/apollo_integration_tests/src/bin/sequencer_node_setup.rs

@@ -265,6 +265,7 @@ impl IntegrationTestManager {
     pub async fn new(
         num_of_consolidated_nodes: usize,
         num_of_distributed_nodes: usize,
+        num_of_hybrid_nodes: usize,
         custom_paths: Option<CustomPaths>,
         test_unique_id: TestIdentifier,
     ) -> Self {
@@ -274,6 +275,7 @@ impl IntegrationTestManager {
             &tx_generator,
             num_of_consolidated_nodes,
             num_of_distributed_nodes,
+            num_of_hybrid_nodes,
             custom_paths,
             test_unique_id,
         )
@@ -856,23 +858,23 @@ async fn get_sequencer_setup_configs(
     // TODO(Tsabary/Nadin): instead of number of nodes, this should be a vector of deployments.
     num_of_consolidated_nodes: usize,
     num_of_distributed_nodes: usize,
+    num_of_hybrid_nodes: usize,
     custom_paths: Option<CustomPaths>,
     test_unique_id: TestIdentifier,
 ) -> (Vec<NodeSetup>, HashSet<usize>) {
     let mut available_ports_generator = AvailablePortsGenerator::new(test_unique_id.into());
 
-    let mut node_component_configs =
-        Vec::with_capacity(num_of_consolidated_nodes + num_of_distributed_nodes);
+    let mut node_component_configs = Vec::with_capacity(
+        num_of_consolidated_nodes + num_of_distributed_nodes + num_of_hybrid_nodes,
+    );
     for _ in 0..num_of_consolidated_nodes {
         node_component_configs.push(create_consolidated_component_configs());
     }
-    // Testing the two various node configurations: distributed and hybrid.
-    // TODO(Tsabary): better handling of the number of each type.
-    for _ in 0..num_of_distributed_nodes / 2 {
+    for _ in 0..num_of_hybrid_nodes {
         node_component_configs
             .push(create_hybrid_component_configs(&mut available_ports_generator));
     }
-    for _ in num_of_distributed_nodes / 2..num_of_distributed_nodes {
+    for _ in 0..num_of_distributed_nodes {
         node_component_configs
             .push(create_distributed_component_configs(&mut available_ports_generator));
     }