Skip to content

Commit 3455605

Browse files
morlov-apexaicarlos-apexchristophebedard
authored
Add tests for rosbag2_performance_benchmarking pkg (ros2#1268)
* Add integration test for rosbag2_performance_benchmarking Co-authored-by: Carlos San Vicente <[email protected]> Signed-off-by: Michael Orlov <[email protected]> * Add underscore for topic index name in WriterBenchmark - Note. It was inconsistency with BenchmarkPublishers Signed-off-by: Michael Orlov <[email protected]> * Enable rosbag2_performance_benchmarking for build and disable for test Signed-off-by: Michael Orlov <[email protected]> * Bump actions-ros-ci to v0.3 Co-authored-by: Christophe Bedard <[email protected]> Signed-off-by: Michael Orlov <[email protected]> Co-authored-by: Christophe Bedard <[email protected]> * Add missing `rosbag2_examples_py` to test and build job Signed-off-by: Michael Orlov <[email protected]> * Add TODOs for code refactoring in future Signed-off-by: Michael Orlov <[email protected]> * Add non default msg_type for camera images and point cloud in automotive.yaml Signed-off-by: Michael Orlov <[email protected]> --------- Signed-off-by: Michael Orlov <[email protected]> Co-authored-by: Carlos San Vicente <[email protected]> Co-authored-by: Christophe Bedard <[email protected]>
1 parent 7b5cd7d commit 3455605

File tree

15 files changed

+588
-33
lines changed

15 files changed

+588
-33
lines changed

‎.github/workflows/test.yml

+50-23
Original file line numberDiff line numberDiff line change
@@ -17,40 +17,67 @@ jobs:
1717
steps:
1818
- name: Build and run tests
1919
id: action-ros-ci
20-
uses: ros-tooling/action-ros-ci@v0.2
20+
uses: ros-tooling/action-ros-ci@v0.3
2121
with:
22-
package-name: |
23-
mcap_vendor
24-
ros2bag
25-
rosbag2
26-
rosbag2_compression
27-
rosbag2_compression_zstd
28-
rosbag2_cpp
29-
rosbag2_examples_cpp
30-
rosbag2_interfaces
31-
rosbag2_py
32-
rosbag2_storage
33-
rosbag2_storage_mcap
34-
rosbag2_storage_sqlite3
35-
rosbag2_test_common
36-
rosbag2_test_msgdefs
37-
rosbag2_tests
38-
rosbag2_transport
39-
shared_queues_vendor
40-
sqlite3_vendor
41-
zstd_vendor
4222
target-ros2-distro: rolling
4323
vcs-repo-file-url: https://raw.githubusercontent.com/ros2/ros2/rolling/ros2.repos
4424
colcon-defaults: |
4525
{
4626
"build": {
4727
"cmake-args": [
48-
"-DCMAKE_CXX_FLAGS=\"-Werror\""
28+
"-DCMAKE_CXX_FLAGS=\"-Werror\"",
29+
"-DBUILD_ROSBAG2_BENCHMARKS=1"
30+
],
31+
"packages-up-to": [
32+
"mcap_vendor",
33+
"ros2bag",
34+
"rosbag2",
35+
"rosbag2_compression",
36+
"rosbag2_compression_zstd",
37+
"rosbag2_cpp",
38+
"rosbag2_examples_cpp",
39+
"rosbag2_examples_py",
40+
"rosbag2_interfaces",
41+
"rosbag2_performance_benchmarking",
42+
"rosbag2_performance_benchmarking_msgs",
43+
"rosbag2_py",
44+
"rosbag2_storage",
45+
"rosbag2_storage_mcap",
46+
"rosbag2_storage_sqlite3",
47+
"rosbag2_test_common",
48+
"rosbag2_test_msgdefs",
49+
"rosbag2_tests",
50+
"rosbag2_transport",
51+
"shared_queues_vendor",
52+
"sqlite3_vendor",
53+
"zstd_vendor"
4954
]
5055
},
5156
"test": {
5257
"ctest-args": ["-LE", "xfail"],
53-
"pytest-args": ["-m", "not xfail"]
58+
"pytest-args": ["-m", "not xfail"],
59+
"packages-select": [
60+
"mcap_vendor",
61+
"ros2bag",
62+
"rosbag2",
63+
"rosbag2_compression",
64+
"rosbag2_compression_zstd",
65+
"rosbag2_cpp",
66+
"rosbag2_examples_cpp",
67+
"rosbag2_examples_py",
68+
"rosbag2_interfaces",
69+
"rosbag2_py",
70+
"rosbag2_storage",
71+
"rosbag2_storage_mcap",
72+
"rosbag2_storage_sqlite3",
73+
"rosbag2_test_common",
74+
"rosbag2_test_msgdefs",
75+
"rosbag2_tests",
76+
"rosbag2_transport",
77+
"shared_queues_vendor",
78+
"sqlite3_vendor",
79+
"zstd_vendor"
80+
]
5481
}
5582
}
5683
- name: Run xfail tests (not required to succeed)

‎rosbag2_performance/rosbag2_performance_benchmarking/CMakeLists.txt

+10
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
cmake_minimum_required(VERSION 3.5)
22
project(rosbag2_performance_benchmarking)
33

4+
option(BUILD_ROSBAG2_BENCHMARKS "Build rosbag2 performance benchmarks" OFF)
5+
46
if(NOT BUILD_ROSBAG2_BENCHMARKS)
57
return()
68
endif()
@@ -93,6 +95,14 @@ install(DIRECTORY
9395
if(BUILD_TESTING)
9496
find_package(ament_lint_auto REQUIRED)
9597
ament_lint_auto_find_test_dependencies()
98+
99+
find_package(launch_testing_ament_cmake)
100+
add_launch_test(
101+
test/benchmark_test.py
102+
TARGET rosbag2_benchmark_test
103+
ENV
104+
TIMEOUT 600
105+
)
96106
endif()
97107

98108
ament_package()

‎rosbag2_performance/rosbag2_performance_benchmarking/config/benchmarks/default_benchmark_producers.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
rosbag2_performance_benchmarking:
2+
benchmark_node:
3+
ros__parameters:
4+
benchmark:
5+
summary_result_file: "results.csv"
6+
bag_root_folder: "/tmp/rosbag2_performance_default"
7+
repeat_each: 1 # How many times to run each configurations (to average results)
8+
no_transport: False # Whether to run storage-only or end-to-end (including transport) benchmark
9+
preserve_bags: False # Whether to leave bag files after experiment (and between runs). Some configurations can take lots of space!
10+
parameters: # Each combination of parameters in this section will be benchmarked
11+
max_cache_size: [100000000, 500000000]
12+
max_bag_size: [0, 1000000000]
13+
compression: [""]
14+
compression_queue_size: [1]
15+
compression_threads: [0]
16+
storage_config_file: [""]

‎rosbag2_performance/rosbag2_performance_benchmarking/config/benchmarks/default_no_transport.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
rosbag2_performance_benchmarking:
2+
benchmark_node:
3+
ros__parameters:
4+
benchmark:
5+
summary_result_file: "results.csv"
6+
bag_root_folder: "/tmp/rosbag2_performance_default"
7+
repeat_each: 1 # How many times to run each configurations (to average results)
8+
no_transport: True # Whether to run storage-only or end-to-end (including transport) benchmark
9+
preserve_bags: False # Whether to leave bag files after experiment (and between runs). Some configurations can take lots of space!
10+
parameters: # Each combination of parameters in this section will be benchmarked
11+
max_cache_size: [100000000, 500000000]
12+
max_bag_size: [0, 1000000000]
13+
compression: [""]
14+
compression_queue_size: [1]
15+
compression_threads: [0]
16+
storage_config_file: [""]

‎rosbag2_performance/rosbag2_performance_benchmarking/config/benchmarks/test/test_benchmark_producers.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
rosbag2_performance_benchmarking:
2+
benchmark_node:
3+
ros__parameters:
4+
benchmark:
5+
summary_result_file: "results.csv"
6+
bag_root_folder: "/tmpfs/rosbag2_performance_test"
7+
repeat_each: 1 # How many times to run each configurations (to average results)
8+
no_transport: False # Whether to run storage-only or end-to-end (including transport) benchmark
9+
preserve_bags: False # Whether to leave bag files after experiment (and between runs). Some configurations can take lots of space!
10+
parameters: # Each combination of parameters in this section will be benchmarked
11+
max_cache_size: [100000000]
12+
max_bag_size: [0]
13+
compression: [""]
14+
compression_queue_size: [1]
15+
compression_threads: [0]
16+
storage_config_file: [""]

‎rosbag2_performance/rosbag2_performance_benchmarking/config/benchmarks/test.yaml ‎rosbag2_performance/rosbag2_performance_benchmarking/config/benchmarks/test/test_no_transport.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,14 @@ rosbag2_performance_benchmarking:
33
ros__parameters:
44
benchmark:
55
summary_result_file: "results.csv"
6-
bag_root_folder: "rosbag2_performance_test_results"
7-
repeat_each: 2 # How many times to run each configurations (to average results)
6+
bag_root_folder: "/tmpfs/rosbag2_performance_test"
7+
repeat_each: 1 # How many times to run each configurations (to average results)
88
no_transport: True # Whether to run storage-only or end-to-end (including transport) benchmark
99
preserve_bags: False # Whether to leave bag files after experiment (and between runs). Some configurations can take lots of space!
1010
parameters: # Each combination of parameters in this section will be benchmarked
11-
max_cache_size: [10000000, 100000000]
11+
max_cache_size: [100000000]
1212
max_bag_size: [0]
13-
compression: ["", "zstd"]
13+
compression: [""]
1414
compression_queue_size: [1]
1515
compression_threads: [0]
16-
storage_config_file: ["", "storage_resilient.yaml"]
16+
storage_config_file: [""]

‎rosbag2_performance/rosbag2_performance_benchmarking/config/producers/100MBs_raw.yaml

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
rosbag2_performance_benchmarking_node:
2+
ros__parameters:
3+
publishers: # publisher_groups parameter needs to include all the subsequent groups
4+
publisher_groups: [ "100MBs" ]
5+
wait_for_subscriptions: True
6+
100MBs:
7+
publishers_count: 20
8+
topic_root: "benchmarking_100"
9+
msg_size_bytes: 200000
10+
msg_count_each: 1000
11+
rate_hz: 25
12+
qos:
13+
qos_durability: "transient_local"

‎rosbag2_performance/rosbag2_performance_benchmarking/config/producers/200MBs_raw.yaml

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
rosbag2_performance_benchmarking_node:
2+
ros__parameters:
3+
publishers: # publisher_groups parameter needs to include all the subsequent groups
4+
publisher_groups: [ "200MBs" ]
5+
wait_for_subscriptions: True
6+
200MBs:
7+
publishers_count: 20
8+
topic_root: "benchmarking_200"
9+
msg_size_bytes: 400000
10+
msg_count_each: 1000
11+
rate_hz: 25
12+
qos:
13+
qos_durability: "transient_local"

‎rosbag2_performance/rosbag2_performance_benchmarking/config/producers/300MBs_raw.yaml

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
rosbag2_performance_benchmarking_node:
2+
ros__parameters:
3+
publishers: # publisher_groups parameter needs to include all the subsequent groups
4+
publisher_groups: [ "300MBs" ]
5+
wait_for_subscriptions: True
6+
300MBs:
7+
publishers_count: 20
8+
topic_root: "benchmarking_300"
9+
msg_size_bytes: 600000
10+
msg_count_each: 1000
11+
rate_hz: 25
12+
qos:
13+
qos_durability: "transient_local"

‎rosbag2_performance/rosbag2_performance_benchmarking/config/producers/automotive.yaml

+3-3
Original file line numberDiff line numberDiff line change
@@ -34,18 +34,18 @@ rosbag2_performance_benchmarking_node:
3434
gps_5KBps:
3535
publishers_count: 1
3636
topic_root: "gps"
37-
msg_type: "sensor_msgs::msg::NavSatFix"
37+
msg_size_bytes: 200
3838
msg_count_each: 250
3939
rate_hz: 25
4040
imu_40KBps:
4141
publishers_count: 1
4242
topic_root: "imu"
43-
msg_type: "sensor_msgs::msg::Imu"
43+
msg_size_bytes: 400
4444
msg_count_each: 1000
4545
rate_hz: 100
4646
ultrasonic_72KBps:
4747
publishers_count: 12
4848
topic_root: "ultrasonic"
49-
msg_type: "sensor_msgs::msg::Range"
49+
msg_size_bytes: 150
5050
msg_count_each: 400
5151
rate_hz: 40

‎rosbag2_performance/rosbag2_performance_benchmarking/config/producers/mixed_110Mbs.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ rosbag2_performance_benchmarking_node:
1313
100Mbs_large:
1414
publishers_count: 1
1515
topic_root: "benchmarking_large"
16-
msg_size_bytes: 10000000
16+
msg_size_bytes: 5000000
1717
msg_count_each: 100
1818
rate_hz: 10
1919
qos: # qos settings are ignored for writer only benchmarking

‎rosbag2_performance/rosbag2_performance_benchmarking/package.xml

+6
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@
2323

2424
<test_depend>ament_lint_auto</test_depend>
2525
<test_depend>ament_lint_common</test_depend>
26+
<test_depend>launch_ros</test_depend>
27+
<test_depend>ros_testing</test_depend>
28+
<test_depend>ros2launch</test_depend>
29+
<test_depend>ros2bag</test_depend>
30+
<test_depend>rosbag2_storage_default_plugins</test_depend>
31+
<test_depend>rosbag2_test_common</test_depend>
2632

2733
<export>
2834
<build_type>ament_cmake</build_type>

‎rosbag2_performance/rosbag2_performance_benchmarking/src/writer_benchmark.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ void WriterBenchmark::create_producers()
155155
" messages before terminating");
156156
const unsigned int queue_max_size = 10;
157157
for (unsigned int i = 0; i < c.count; ++i) {
158-
std::string topic = c.topic_root + std::to_string(i);
158+
std::string topic = c.topic_root + "_" + std::to_string(i + 1);
159159
auto queue = std::make_shared<ByteMessageQueue>(queue_max_size, topic);
160160
queues_.push_back(queue);
161161
producers_.push_back(

0 commit comments

Comments
 (0)