From 19220305069eee66ce642ece65062be3dae46f86 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Tue, 3 Jun 2025 10:47:34 +0100 Subject: [PATCH 01/46] doc tweak --- README.md | 9 ++++++--- questdb-rs/README.md | 8 +++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d649e63d..96c32ad5 100644 --- a/README.md +++ b/README.md @@ -48,9 +48,12 @@ The library supports the following ILP protocol versions. These protocol versions are supported over both HTTP and TCP. -If you use HTTP, the library will automatically detect the server's -latest supported protocol version and use it. If you use TCP, you can specify the -`protocol_version=N` parameter when constructing the `Sender` object. +* If you use HTTP and `protocol_version=auto` or unset, the library will + automatically detect the server's + latest supported protocol version and use it (recommended). +* If you use TCP, you can specify the + `protocol_version=N` parameter when constructing the `Sender` object + (TCP defaults to `protocol_version=1`). | Version | Description | Server Comatibility | | ------- | ------------------------------------------------------- | --------------------- | diff --git a/questdb-rs/README.md b/questdb-rs/README.md index e0ef8c93..f8a3db25 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -15,9 +15,11 @@ The library supports the following ILP protocol versions. These protocol versions are supported over both HTTP and TCP. -If you use HTTP, the library will automatically detect the server's -latest supported protocol version and use it. If you use TCP, you can specify the -`protocol_version=N` parameter when constructing the `Sender` object. +* If you use HTTP, the library will automatically detect the server's + latest supported protocol version and use it (recommended). +* If you use TCP, you can specify the + `protocol_version=N` parameter when constructing the `Sender` object + (TCP defaults to `protocol_version=1`). | Version | Description | Server Comatibility | | ------- | ------------------------------------------------------- | --------------------- | From f5dcf8677446c1f096c7f328937f5868bf2d2ef8 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Tue, 3 Jun 2025 10:51:12 +0100 Subject: [PATCH 02/46] fixed a typo --- README.md | 2 +- questdb-rs/README.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 96c32ad5..306ae876 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ These protocol versions are supported over both HTTP and TCP. `protocol_version=N` parameter when constructing the `Sender` object (TCP defaults to `protocol_version=1`). -| Version | Description | Server Comatibility | +| Version | Description | Server Compatibility | | ------- | ------------------------------------------------------- | --------------------- | | **1** | Over HTTP it's compatible InfluxDB Line Protocol (ILP) | All QuestDB versions | | **2** | 64-bit floats sent as binary, adds n-dimentional arrays | 8.4.0+ (2023-10-30) | diff --git a/questdb-rs/README.md b/questdb-rs/README.md index f8a3db25..61eb85d3 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -15,13 +15,14 @@ The library supports the following ILP protocol versions. These protocol versions are supported over both HTTP and TCP. -* If you use HTTP, the library will automatically detect the server's +* If you use HTTP and `protocol_version=auto` or unset, the library will + automatically detect the server's latest supported protocol version and use it (recommended). * If you use TCP, you can specify the `protocol_version=N` parameter when constructing the `Sender` object (TCP defaults to `protocol_version=1`). -| Version | Description | Server Comatibility | +| Version | Description | Server Compatibility | | ------- | ------------------------------------------------------- | --------------------- | | **1** | Over HTTP it's compatible InfluxDB Line Protocol (ILP) | All QuestDB versions | | **2** | 64-bit floats sent as binary, adds n-dimentional arrays | 8.4.0+ (2023-10-30) | From 8464f738ba09f3664a64c9d1cfa44c7715283762 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Tue, 3 Jun 2025 11:49:01 +0100 Subject: [PATCH 03/46] consolidating error codes --- include/questdb/ingress/line_sender.h | 11 ++------ questdb-rs-ffi/src/lib.rs | 20 +++------------ questdb-rs-ffi/src/ndarr.rs | 26 +++++++++---------- questdb-rs/src/error.rs | 10 ++------ questdb-rs/src/ingress/mod.rs | 4 +-- questdb-rs/src/ingress/ndarr.rs | 36 +++++++++++++-------------- questdb-rs/src/tests/ndarr.rs | 8 +++--- 7 files changed, 44 insertions(+), 71 deletions(-) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 3efb424b..7304c4a4 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -78,15 +78,8 @@ typedef enum line_sender_error_code /** Bad configuration. */ line_sender_error_config_error, - /** Currently, only arrays with a maximum 32 dimensions are supported. */ - line_sender_error_array_large_dim, - - /** ArrayView internal error, such as failure to get the size of a valid - * dimension. */ - line_sender_error_array_view_internal_error, - - /** Write arrayView to sender buffer error. */ - line_sender_error_array_view_write_to_buffer_error, + /** There was an error serializing an array. */ + line_sender_error_array_error, /** Line sender protocol version error. */ line_sender_error_protocol_version_error, diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 150b7412..eef6dc74 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -140,14 +140,8 @@ pub enum line_sender_error_code { /// Bad configuration. line_sender_error_config_error, - /// Currently, only arrays with a maximum 32 dimensions are supported. - line_sender_error_array_large_dim, - - /// ArrayView internal error, such as failure to get the size of a valid dimension. - line_sender_error_array_view_internal_error, - - /// Write arrayView to sender buffer error. - line_sender_error_array_view_write_to_buffer_error, + /// There was an error serializing an array. + line_sender_error_array_error, /// Line sender protocol version error. line_sender_error_protocol_version_error, @@ -175,15 +169,7 @@ impl From for line_sender_error_code { line_sender_error_code::line_sender_error_server_flush_error } ErrorCode::ConfigError => line_sender_error_code::line_sender_error_config_error, - ErrorCode::ArrayHasTooManyDims => { - line_sender_error_code::line_sender_error_array_large_dim - } - ErrorCode::ArrayViewError => { - line_sender_error_code::line_sender_error_array_view_internal_error - } - ErrorCode::ArrayWriteToBufferError => { - line_sender_error_code::line_sender_error_array_view_write_to_buffer_error - } + ErrorCode::ArrayError => line_sender_error_code::line_sender_error_array_error, ErrorCode::ProtocolVersionError => { line_sender_error_code::line_sender_error_protocol_version_error } diff --git a/questdb-rs-ffi/src/ndarr.rs b/questdb-rs-ffi/src/ndarr.rs index ed3de205..0c7b12d6 100644 --- a/questdb-rs-ffi/src/ndarr.rs +++ b/questdb-rs-ffi/src/ndarr.rs @@ -71,7 +71,7 @@ where fn dim(&self, index: usize) -> Result { if index >= self.dims { return Err(fmt_error!( - ArrayViewError, + ArrayError, "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", index, self.dims @@ -151,13 +151,13 @@ where ) -> Result { if dims == 0 { return Err(fmt_error!( - ArrayViewError, + ArrayError, "Zero-dimensional arrays are not supported", )); } if data_len > MAX_ARRAY_BUFFER_SIZE { return Err(fmt_error!( - ArrayViewError, + ArrayError, "Array buffer size too big: {}, maximum: {}", data_len, MAX_ARRAY_BUFFER_SIZE @@ -168,12 +168,12 @@ where .iter() .try_fold(std::mem::size_of::(), |acc, &dim| { acc.checked_mul(dim) - .ok_or_else(|| fmt_error!(ArrayViewError, "Array buffer size too big")) + .ok_or_else(|| fmt_error!(ArrayError, "Array buffer size too big")) })?; if size != data_len { return Err(fmt_error!( - ArrayViewError, + ArrayError, "Array buffer length mismatch (actual: {}, expected: {})", data_len, size @@ -299,7 +299,7 @@ mod tests { if bytes.len() != expect_size { return Err(fmt_error!( - ArrayWriteToBufferError, + ArrayError, "Array write buffer length mismatch (actual: {}, expected: {})", expect_size, bytes.len() @@ -308,7 +308,7 @@ mod tests { if buf.len() < bytes.len() { return Err(fmt_error!( - ArrayWriteToBufferError, + ArrayError, "Buffer capacity {} < required {}", buf.len(), bytes.len() @@ -334,7 +334,7 @@ mod tests { } if total_len != expect_size { return Err(fmt_error!( - ArrayWriteToBufferError, + ArrayError, "Array write buffer length mismatch (actual: {}, expected: {})", total_len, expect_size @@ -445,7 +445,7 @@ mod tests { ) }; let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert_eq!(err.code(), ErrorCode::ArrayError); assert!(err.msg().contains("Array buffer size too big")); Ok(()) } @@ -464,7 +464,7 @@ mod tests { ) }; let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert_eq!(err.code(), ErrorCode::ArrayError); assert!(err .msg() .contains("Array buffer length mismatch (actual: 8, expected: 16)")); @@ -481,7 +481,7 @@ mod tests { }; let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert_eq!(err.code(), ErrorCode::ArrayError); assert!(err .msg() .contains("Array buffer length mismatch (actual: 24, expected: 16)")); @@ -502,7 +502,7 @@ mod tests { ) }; let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert_eq!(err.code(), ErrorCode::ArrayError); assert!(err .msg() .contains("Array buffer length mismatch (actual: 8, expected: 16)")); @@ -519,7 +519,7 @@ mod tests { }; let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert_eq!(err.code(), ErrorCode::ArrayError); assert!(err .msg() .contains("Array buffer length mismatch (actual: 24, expected: 16)")); diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index 0c32f48a..b77d6334 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -73,14 +73,8 @@ pub enum ErrorCode { /// Bad configuration. ConfigError, - /// Array has too many dims. Currently, only arrays with a maximum [`crate::ingress::MAX_ARRAY_DIMS`] dimensions are supported. - ArrayHasTooManyDims, - - /// Array view internal error. - ArrayViewError, - - /// Array write to buffer error. - ArrayWriteToBufferError, + /// There was an error serializing an array. + ArrayError, /// Validate protocol version error. ProtocolVersionError, diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index dfecc7b6..29881965 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -1144,7 +1144,7 @@ impl Buffer { let ndim = view.ndim(); if ndim == 0 { return Err(error::fmt!( - ArrayViewError, + ArrayError, "Zero-dimensional arrays are not supported", )); } @@ -1152,7 +1152,7 @@ impl Buffer { // check dimension less equal than max dims if MAX_ARRAY_DIMS < ndim { return Err(error::fmt!( - ArrayHasTooManyDims, + ArrayError, "Array dimension mismatch: expected at most {} dimensions, but got {}", MAX_ARRAY_DIMS, ndim diff --git a/questdb-rs/src/ingress/ndarr.rs b/questdb-rs/src/ingress/ndarr.rs index c793094b..0a1374c0 100644 --- a/questdb-rs/src/ingress/ndarr.rs +++ b/questdb-rs/src/ingress/ndarr.rs @@ -64,7 +64,7 @@ where if bytes.len() != expect_size { return Err(error::fmt!( - ArrayWriteToBufferError, + ArrayError, "Array write buffer length mismatch (actual: {}, expected: {})", expect_size, bytes.len() @@ -73,7 +73,7 @@ where if buf.len() < bytes.len() { return Err(error::fmt!( - ArrayWriteToBufferError, + ArrayError, "Buffer capacity {} < required {}", buf.len(), bytes.len() @@ -99,7 +99,7 @@ where } if total_len != expect_size { return Err(error::fmt!( - ArrayWriteToBufferError, + ArrayError, "Array write buffer length mismatch (actual: {}, expected: {})", total_len, expect_size @@ -119,7 +119,7 @@ where let dim = array.dim(dim_index)?; if dim > MAX_ARRAY_DIM_LEN { return Err(error::fmt!( - ArrayViewError, + ArrayError, "dimension length out of range: dim {}, dim length {}, max length {}", dim_index, dim, @@ -132,7 +132,7 @@ where if size > MAX_ARRAY_BUFFER_SIZE { return Err(error::fmt!( - ArrayViewError, + ArrayError, "Array buffer size too big: {}, maximum: {}", size, MAX_ARRAY_BUFFER_SIZE @@ -177,7 +177,7 @@ impl NdArrayView for Vec { Ok(self.len()) } else { Err(error::fmt!( - ArrayViewError, + ArrayError, "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", idx, 1 @@ -210,7 +210,7 @@ impl NdArrayView for [T; N] { Ok(N) } else { Err(error::fmt!( - ArrayViewError, + ArrayError, "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", idx, 1 @@ -244,7 +244,7 @@ impl NdArrayView for &[T] { Ok(self.len()) } else { Err(error::fmt!( - ArrayViewError, + ArrayError, "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", idx, 1 @@ -278,12 +278,12 @@ impl NdArrayView for Vec> { 1 => { let dim1 = self.first().map_or(0, |v| v.len()); if self.as_slice().iter().any(|v2| v2.len() != dim1) { - return Err(error::fmt!(ArrayViewError, "Irregular array shape")); + return Err(error::fmt!(ArrayError, "Irregular array shape")); } Ok(dim1) } _ => Err(error::fmt!( - ArrayViewError, + ArrayError, "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", idx, 2 @@ -316,7 +316,7 @@ impl NdArrayView for [[T; M] 0 => Ok(N), 1 => Ok(M), _ => Err(error::fmt!( - ArrayViewError, + ArrayError, "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", idx, 2 @@ -350,7 +350,7 @@ impl NdArrayView for &[[T; M]] { 0 => Ok(self.len()), 1 => Ok(M), _ => Err(error::fmt!( - ArrayViewError, + ArrayError, "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", idx, 2 @@ -384,7 +384,7 @@ impl NdArrayView for Vec>> { 1 => { let dim1 = self.first().map_or(0, |v| v.len()); if self.as_slice().iter().any(|v2| v2.len() != dim1) { - return Err(error::fmt!(ArrayViewError, "Irregular array shape")); + return Err(error::fmt!(ArrayError, "Irregular array shape")); } Ok(dim1) } @@ -400,12 +400,12 @@ impl NdArrayView for Vec>> { .flat_map(|v2| v2.as_slice().iter()) .any(|v3| v3.len() != dim2) { - return Err(error::fmt!(ArrayViewError, "Irregular array shape")); + return Err(error::fmt!(ArrayError, "Irregular array shape")); } Ok(dim2) } _ => Err(error::fmt!( - ArrayViewError, + ArrayError, "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", idx, 3 @@ -441,7 +441,7 @@ impl NdArrayVie 1 => Ok(N), 2 => Ok(M), _ => Err(error::fmt!( - ArrayViewError, + ArrayError, "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", idx, 3 @@ -475,7 +475,7 @@ impl NdArrayView for &[[[T; 1 => Ok(N), 2 => Ok(M), _ => Err(error::fmt!( - ArrayViewError, + ArrayError, "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", idx, 3 @@ -521,7 +521,7 @@ where Ok(self.len_of(Axis(index))) } else { Err(error::fmt!( - ArrayViewError, + ArrayError, "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", index, 3 diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 659f8ab4..1e5e3bb9 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -390,7 +390,7 @@ fn test_build_in_2d_vec_irregular_shape() -> TestResult { buffer.table("my_test")?; let result = buffer.column_arr("arr", &irregular_vec); let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert_eq!(err.code(), ErrorCode::ArrayError); assert!(err.msg().contains("Irregular array shape")); Ok(()) } @@ -688,12 +688,12 @@ fn test_build_in_3d_vec_irregular_shape() -> TestResult { buffer.table("my_test")?; let result = buffer.column_arr("arr", &irregular1); let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert_eq!(err.code(), ErrorCode::ArrayError); assert!(err.msg().contains("Irregular array shape")); let result = buffer.column_arr("arr", &irregular2); let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayViewError); + assert_eq!(err.code(), ErrorCode::ArrayError); assert!(err.msg().contains("Irregular array shape")); Ok(()) } @@ -900,6 +900,6 @@ fn test_buffer_write_ndarray_max_dimensions() -> TestResult { let result = buffer.column_arr("invalid", &array_invalid.view()); assert!(result.is_err()); let err = result.unwrap_err(); - assert_eq!(err.code(), ErrorCode::ArrayHasTooManyDims); + assert_eq!(err.code(), ErrorCode::ArrayError); Ok(()) } From 9d855502853be5235c1f8d2ffb156b790f4ac084 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Tue, 3 Jun 2025 12:04:37 +0100 Subject: [PATCH 04/46] fixed C++ formatting --- cpp_test/test_line_sender.cpp | 38 +++++++++++++++++------------------ 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index e2507797..502198d4 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -178,29 +178,27 @@ TEST_CASE("line_sender c api basics") 2.7, 48121.5, 4.3}; - CHECK( - ::line_sender_buffer_column_f64_arr_byte_strides( - buffer, - arr_name, - rank, - shape, - strides, - reinterpret_cast(arr_data.data()), - sizeof(arr_data), - &err)); + CHECK(::line_sender_buffer_column_f64_arr_byte_strides( + buffer, + arr_name, + rank, + shape, + strides, + reinterpret_cast(arr_data.data()), + sizeof(arr_data), + &err)); line_sender_column_name arr_name2 = QDB_COLUMN_NAME_LITERAL("a2"); intptr_t elem_strides[] = {6, 2, 1}; - CHECK( - ::line_sender_buffer_column_f64_arr_elem_strides( - buffer, - arr_name2, - rank, - shape, - elem_strides, - reinterpret_cast(arr_data.data()), - sizeof(arr_data), - &err)); + CHECK(::line_sender_buffer_column_f64_arr_elem_strides( + buffer, + arr_name2, + rank, + shape, + elem_strides, + reinterpret_cast(arr_data.data()), + sizeof(arr_data), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); CHECK(::line_sender_buffer_size(buffer) == 266); From dee903c21901e2b1aea983ba2a440337c1e7a236 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Tue, 3 Jun 2025 12:14:40 +0100 Subject: [PATCH 05/46] fix to run against merged array branch --- ci/run_tests_pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 03b8d129..f7fcc22b 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -56,7 +56,7 @@ stages: displayName: "Build Rust examples" ############################# temp for test begin ##################### - script: | - git clone -b nd_arr --depth 1 https://github.com/questdb/questdb.git ./questdb_nd_arr + git clone -b master --depth 1 https://github.com/questdb/questdb.git ./questdb_nd_arr displayName: git clone questdb - task: Maven@3 displayName: "Compile QuestDB" From 041776490cc65d2d92380be89e549d65e98eddda Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Tue, 3 Jun 2025 12:34:21 +0100 Subject: [PATCH 06/46] fixed TestVsQuestDBMaster CI job --- ci/run_tests_pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index f7fcc22b..65070c6c 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -126,7 +126,7 @@ stages: submodules: false - template: compile.yaml - script: | - git clone -b nd_arr --depth 1 https://github.com/questdb/questdb.git + git clone -b master --depth 1 https://github.com/questdb/questdb.git displayName: git clone questdb - task: Maven@3 displayName: "Compile QuestDB" From 19c47cbf60a85ab6626924ded53008af88c8586c Mon Sep 17 00:00:00 2001 From: victor Date: Tue, 3 Jun 2025 22:51:13 +0800 Subject: [PATCH 07/46] add tests for `max_name_len` and fix one bug. --- questdb-rs/src/ingress/http.rs | 8 ++-- questdb-rs/src/ingress/mod.rs | 4 +- questdb-rs/src/tests/http.rs | 87 +++++++++++++++++++++++++++++++--- questdb-rs/src/tests/mock.rs | 8 +++- system_test/test.py | 6 ++- 5 files changed, 97 insertions(+), 16 deletions(-) diff --git a/questdb-rs/src/ingress/http.rs b/questdb-rs/src/ingress/http.rs index 68d5e4a6..e340235b 100644 --- a/questdb-rs/src/ingress/http.rs +++ b/questdb-rs/src/ingress/http.rs @@ -23,7 +23,6 @@ ******************************************************************************/ use super::conf::ConfigSetting; -use super::MAX_NAME_LEN_DEFAULT; use crate::error::fmt; use crate::{error, Error}; use base64ct::Base64; @@ -441,6 +440,7 @@ pub(super) fn http_send_with_retries( pub(super) fn read_server_settings( state: &HttpHandlerState, settings_url: &str, + default_max_name_len: usize, ) -> Result<(Vec, usize), Error> { let default_protocol_version = ProtocolVersion::V1; @@ -455,7 +455,7 @@ pub(super) fn read_server_settings( let status = res.status(); _ = res.into_body().read_to_vec(); if status.as_u16() == 404 { - return Ok((vec![default_protocol_version], MAX_NAME_LEN_DEFAULT)); + return Ok((vec![default_protocol_version], default_max_name_len)); } return Err(fmt!( ProtocolVersionError, @@ -471,7 +471,7 @@ pub(super) fn read_server_settings( let e = match err { ureq::Error::StatusCode(code) => { if code == 404 { - return Ok((vec![default_protocol_version], MAX_NAME_LEN_DEFAULT)); + return Ok((vec![default_protocol_version], default_max_name_len)); } else { fmt!( ProtocolVersionError, @@ -528,7 +528,7 @@ pub(super) fn read_server_settings( .get("config") .and_then(|v| v.get("cairo.max.file.name.length")) .and_then(|v| v.as_u64()) - .unwrap_or(MAX_NAME_LEN_DEFAULT as u64) as usize; + .unwrap_or(default_max_name_len as u64) as usize; Ok((support_versions, max_name_length)) } else { Err(error::fmt!( diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 29881965..db7a9c74 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -57,7 +57,7 @@ use ring::{ signature::{EcdsaKeyPair, ECDSA_P256_SHA256_FIXED_SIGNING}, }; -pub(crate) const MAX_NAME_LEN_DEFAULT: usize = 127; +const MAX_NAME_LEN_DEFAULT: usize = 127; /// The maximum allowed dimensions for arrays. pub const MAX_ARRAY_DIMS: usize = 32; @@ -2618,7 +2618,7 @@ impl SenderBuilder { self.port.deref() ); let (protocol_versions, server_max_name_len) = - read_server_settings(http_state, settings_url)?; + read_server_settings(http_state, settings_url, max_name_len)?; max_name_len = server_max_name_len; if protocol_versions.contains(&ProtocolVersion::V2) { ProtocolVersion::V2 diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index 172267e6..d66242db 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -701,10 +701,12 @@ fn test_transactional( fn _test_sender_auto_detect_protocol_version( supported_versions: Option>, expect_version: ProtocolVersion, + max_name_len: usize, + expect_max_name_len: usize, ) -> TestResult { let supported_versions1 = supported_versions.clone(); let mut server = MockServer::new()? - .configure_settings_response(supported_versions.as_deref().unwrap_or(&[])); + .configure_settings_response(supported_versions.as_deref().unwrap_or(&[]), max_name_len); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result { @@ -735,6 +737,7 @@ fn _test_sender_auto_detect_protocol_version( let mut sender = sender_builder.build()?; assert_eq!(sender.protocol_version(), expect_version); + assert_eq!(sender.max_name_len(), expect_max_name_len); let mut buffer = sender.new_buffer(); buffer .table("test")? @@ -749,32 +752,32 @@ fn _test_sender_auto_detect_protocol_version( #[test] fn test_sender_auto_protocol_version_basic() -> TestResult { - _test_sender_auto_detect_protocol_version(Some(vec![1, 2]), ProtocolVersion::V2) + _test_sender_auto_detect_protocol_version(Some(vec![1, 2]), ProtocolVersion::V2, 130, 130) } #[test] fn test_sender_auto_protocol_version_old_server1() -> TestResult { - _test_sender_auto_detect_protocol_version(Some(vec![]), ProtocolVersion::V1) + _test_sender_auto_detect_protocol_version(Some(vec![]), ProtocolVersion::V1, 0, 127) } #[test] fn test_sender_auto_protocol_version_old_server2() -> TestResult { - _test_sender_auto_detect_protocol_version(None, ProtocolVersion::V1) + _test_sender_auto_detect_protocol_version(None, ProtocolVersion::V1, 0, 127) } #[test] fn test_sender_auto_protocol_version_only_v1() -> TestResult { - _test_sender_auto_detect_protocol_version(Some(vec![1]), ProtocolVersion::V1) + _test_sender_auto_detect_protocol_version(Some(vec![1]), ProtocolVersion::V1, 127, 127) } #[test] fn test_sender_auto_protocol_version_only_v2() -> TestResult { - _test_sender_auto_detect_protocol_version(Some(vec![2]), ProtocolVersion::V2) + _test_sender_auto_detect_protocol_version(Some(vec![2]), ProtocolVersion::V2, 127, 127) } #[test] fn test_sender_auto_protocol_version_unsupported_client() -> TestResult { - let mut server = MockServer::new()?.configure_settings_response(&[3, 4]); + let mut server = MockServer::new()?.configure_settings_response(&[3, 4], 127); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; @@ -792,6 +795,76 @@ fn test_sender_auto_protocol_version_unsupported_client() -> TestResult { Ok(()) } +#[test] +fn test_sender_short_max_name_len() -> TestResult { + _test_sender_max_name_len(4, 4, 0) +} + +#[test] +fn test_sender_specify_max_name_len_with_response() -> TestResult { + _test_sender_max_name_len(4, 4, 127) +} + +#[test] +fn test_sender_long_max_name_len() -> TestResult { + _test_sender_max_name_len(130, 130, 0) +} + +#[test] +fn test_sender_specify_max_name_len_without_response() -> TestResult { + _test_sender_max_name_len(0, 16, 16) +} + +#[test] +fn test_sender_default_max_name_len() -> TestResult { + _test_sender_max_name_len(0, 127, 0) +} + +fn _test_sender_max_name_len( + response_max_name_len: usize, + expect_max_name_len: usize, + sender_specify_max_name_len: usize, +) -> TestResult { + let mut server = MockServer::new()?; + if response_max_name_len != 0 { + server = server.configure_settings_response(&[1, 2], response_max_name_len); + } + + let mut sender_builder = server.lsb_http(); + if sender_specify_max_name_len != 0 { + sender_builder = sender_builder.max_name_len(sender_specify_max_name_len)?; + } + let server_thread = std::thread::spawn(move || -> io::Result { + server.accept()?; + match response_max_name_len { + 0 => server.send_http_response_q( + HttpResponse::empty() + .with_status(404, "Not Found") + .with_header("content-type", "text/plain") + .with_body_str("Not Found"), + )?, + _ => server.send_settings_response()?, + } + Ok(server) + }); + let sender = sender_builder.build()?; + assert_eq!(sender.max_name_len(), expect_max_name_len); + let mut buffer = sender.new_buffer(); + let name = "a name too long"; + if expect_max_name_len < name.len() { + assert_err_contains( + buffer.table(name), + ErrorCode::InvalidName, + r#"Bad name: "a name too long": Too long (max 4 characters)"#, + ); + } else { + assert!(buffer.table(name).is_ok()); + } + // We keep the server around til the end of the test to ensure that the response is fully received. + _ = server_thread.join().unwrap()?; + Ok(()) +} + #[test] fn test_buffer_protocol_version1_not_support_array() -> TestResult { let mut buffer = Buffer::new(ProtocolVersion::V1); diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index caa407a6..d20d3456 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -310,7 +310,11 @@ impl MockServer { } #[cfg(feature = "ilp-over-http")] - pub fn configure_settings_response(mut self, supported_versions: &[u16]) -> Self { + pub fn configure_settings_response( + mut self, + supported_versions: &[u16], + max_name_len: usize, + ) -> Self { if supported_versions.is_empty() { self.settings_response = serde_json::json!({"version": "8.1.2"}); } else { @@ -320,7 +324,7 @@ impl MockServer { "ilp.proto.transports":["tcp", "http"], "posthog.enabled":false, "posthog.api.key":null, - "cairo.max.file.name.length":127}, + "cairo.max.file.name.length": max_name_len}, "preferences.version":0, "preferences":{}} ); diff --git a/system_test/test.py b/system_test/test.py index e7d60dbc..929dd766 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -54,7 +54,7 @@ BUILD_MODE = None # The first QuestDB version that supports array types. -FIRST_ARRAYS_RELEASE = (8, 3, 1) +FIRST_ARRAYS_RELEASE = (8, 3, 3) def retry_check_table(*args, **kwargs): @@ -137,6 +137,10 @@ def _expect_eventual_disconnect(self, sender): .at_now()) sender.flush() + def test_default_max_name_len(self): + with self._mk_linesender() as sender: + self.assertEqual(sender.max_name_len, 127) + def test_insert_three_rows(self): table_name = uuid.uuid4().hex pending = None From d8843d4a28a9d0f8b063dd0d20c51f50b0a792d1 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 4 Jun 2025 13:14:45 +0800 Subject: [PATCH 08/46] add c-major array layout c api. --- CMakeLists.txt | 7 + cpp_test/test_line_sender.cpp | 25 +- .../line_sender_c_example_array_c_major.c | 96 +++++++ ..._sender_cpp_example_array_byte_strides.cpp | 3 +- .../line_sender_cpp_example_array_c_major.cpp | 62 +++++ ..._sender_cpp_example_array_elem_strides.cpp | 3 +- include/questdb/ingress/line_sender.h | 24 ++ include/questdb/ingress/line_sender.hpp | 70 ++++- questdb-rs-ffi/src/lib.rs | 41 +++ questdb-rs-ffi/src/ndarr.rs | 256 ++++++++++++++++-- system_test/questdb_line_sender.py | 40 ++- system_test/test.py | 6 + 12 files changed, 581 insertions(+), 52 deletions(-) create mode 100644 examples/line_sender_c_example_array_c_major.c create mode 100644 examples/line_sender_cpp_example_array_c_major.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b36780e..1fcea964 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -107,6 +107,10 @@ if (QUESTDB_TESTS_AND_EXAMPLES) line_sender_c_example_array_elem_strides examples/concat.c examples/line_sender_c_example_array_elem_strides.c) + compile_example( + line_sender_c_example_array_c_major + examples/concat.c + examples/line_sender_c_example_array_c_major.c) compile_example( line_sender_c_example_auth examples/concat.c @@ -141,6 +145,9 @@ if (QUESTDB_TESTS_AND_EXAMPLES) compile_example( line_sender_cpp_example_auth examples/line_sender_cpp_example_auth.cpp) + compile_example( + line_sender_cpp_example_array_c_major + examples/line_sender_cpp_example_array_c_major.cpp) compile_example( line_sender_cpp_example_tls_ca examples/line_sender_cpp_example_tls_ca.cpp) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 502198d4..9d80a8c9 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -199,15 +199,26 @@ TEST_CASE("line_sender c api basics") reinterpret_cast(arr_data.data()), sizeof(arr_data), &err)); + line_sender_column_name arr_name3 = QDB_COLUMN_NAME_LITERAL("a3"); + CHECK( + ::line_sender_buffer_column_f64_arr_c_major( + buffer, + arr_name3, + rank, + shape, + reinterpret_cast(arr_data.data()), + sizeof(arr_data), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); - CHECK(::line_sender_buffer_size(buffer) == 266); + CHECK(::line_sender_buffer_size(buffer) == 382); CHECK(::line_sender_flush(sender, buffer, &err)); ::line_sender_buffer_free(buffer); CHECK(server.recv() == 1); std::string expect{"test,t1=v1 f1=="}; push_double_to_buffer(expect, 0.5).append(",a1=="); push_double_arr_to_buffer(expect, arr_data, 3, shape).append(",a2=="); + push_double_arr_to_buffer(expect, arr_data, 3, shape).append(",a3=="); push_double_arr_to_buffer(expect, arr_data, 3, shape).append(" 10000000\n"); CHECK(server.msgs(0) == expect); } @@ -282,18 +293,24 @@ TEST_CASE("line_sender c++ api basics") .symbol("t1", "v1") .symbol("t2", "") .column("f1", 0.5) - .column("a1", rank, shape, strides, arr_data) - .column("a2", rank, shape, elem_strides, arr_data) + .column( + "a1", rank, shape, strides, arr_data) + .column( + "a2", rank, shape, elem_strides, arr_data) + .column( + "a3", rank, shape, {}, arr_data) .at(questdb::ingress::timestamp_nanos{10000000}); CHECK(server.recv() == 0); - CHECK(buffer.size() == 270); + CHECK(buffer.size() == 386); sender.flush(buffer); CHECK(server.recv() == 1); std::string expect{"test,t1=v1,t2= f1=="}; push_double_to_buffer(expect, 0.5).append(",a1=="); push_double_arr_to_buffer(expect, arr_data, 3, shape.data()) .append(",a2=="); + push_double_arr_to_buffer(expect, arr_data, 3, shape.data()) + .append(",a3=="); push_double_arr_to_buffer(expect, arr_data, 3, shape.data()) .append(" 10000000\n"); CHECK(server.msgs(0) == expect); diff --git a/examples/line_sender_c_example_array_c_major.c b/examples/line_sender_c_example_array_c_major.c new file mode 100644 index 00000000..806aa140 --- /dev/null +++ b/examples/line_sender_c_example_array_c_major.c @@ -0,0 +1,96 @@ +#include +#include +#include +#include +#include "concat.h" + +static bool example(const char* host, const char* port) +{ + line_sender_error* err = NULL; + line_sender* sender = NULL; + line_sender_buffer* buffer = NULL; + char* conf_str = concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); + if (!conf_str) + { + fprintf(stderr, "Could not concatenate configuration string.\n"); + return false; + } + + line_sender_utf8 conf_str_utf8 = {0, NULL}; + if (!line_sender_utf8_init( + &conf_str_utf8, strlen(conf_str), conf_str, &err)) + goto on_error; + + sender = line_sender_from_conf(conf_str_utf8, &err); + if (!sender) + goto on_error; + + free(conf_str); + conf_str = NULL; + + buffer = line_sender_buffer_new_for_sender(sender); + line_sender_buffer_reserve(buffer, 64 * 1024); + + line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("market_orders_c_major"); + line_sender_column_name symbol_col = QDB_COLUMN_NAME_LITERAL("symbol"); + line_sender_column_name book_col = QDB_COLUMN_NAME_LITERAL("order_book"); + + if (!line_sender_buffer_table(buffer, table_name, &err)) + goto on_error; + + line_sender_utf8 symbol_val = QDB_UTF8_LITERAL("BTC-USD"); + if (!line_sender_buffer_symbol(buffer, symbol_col, symbol_val, &err)) + goto on_error; + + size_t array_rank = 3; + uintptr_t array_shape[] = {2, 3, 2}; + double array_data[] = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + + if (!line_sender_buffer_column_f64_arr_c_major( + buffer, + book_col, + array_rank, + array_shape, + (const uint8_t*)array_data, + sizeof(array_data), + &err)) + goto on_error; + + if (!line_sender_buffer_at_nanos(buffer, line_sender_now_nanos(), &err)) + goto on_error; + + if (!line_sender_flush(sender, buffer, &err)) + goto on_error; + + line_sender_close(sender); + return true; + +on_error:; + size_t err_len = 0; + const char* err_msg = line_sender_error_msg(err, &err_len); + fprintf(stderr, "Error: %.*s\n", (int)err_len, err_msg); + free(conf_str); + line_sender_error_free(err); + line_sender_buffer_free(buffer); + line_sender_close(sender); + return false; +} + +int main(int argc, const char* argv[]) +{ + const char* host = (argc >= 2) ? argv[1] : "localhost"; + const char* port = (argc >= 3) ? argv[2] : "9009"; + return !example(host, port); +} diff --git a/examples/line_sender_cpp_example_array_byte_strides.cpp b/examples/line_sender_cpp_example_array_byte_strides.cpp index 88de8a39..58505fbc 100644 --- a/examples/line_sender_cpp_example_array_byte_strides.cpp +++ b/examples/line_sender_cpp_example_array_byte_strides.cpp @@ -36,7 +36,8 @@ static bool array_example(std::string_view host, std::string_view port) questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_col, "BTC-USD"_utf8) - .column(book_col, 3, shape, strides, arr_data) + .column( + book_col, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); sender.flush(buffer); return true; diff --git a/examples/line_sender_cpp_example_array_c_major.cpp b/examples/line_sender_cpp_example_array_c_major.cpp new file mode 100644 index 00000000..c5c44438 --- /dev/null +++ b/examples/line_sender_cpp_example_array_c_major.cpp @@ -0,0 +1,62 @@ +#include +#include +#include + +using namespace std::literals::string_view_literals; +using namespace questdb::ingress::literals; + +static bool array_example(std::string_view host, std::string_view port) +{ + try + { + auto sender = questdb::ingress::line_sender::from_conf( + "tcp::addr=" + std::string{host} + ":" + std::string{port} + + ";protocol_version=2;"); + + const auto table_name = "cpp_market_orders_c_major"_tn; + const auto symbol_col = "symbol"_cn; + const auto book_col = "order_book"_cn; + size_t rank = 3; + std::vector shape{2, 3, 2}; + std::array arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) + .symbol(symbol_col, "BTC-USD"_utf8) + .column( + book_col, 3, shape, {}, arr_data) + .at(questdb::ingress::timestamp_nanos::now()); + sender.flush(buffer); + return true; + } + catch (const questdb::ingress::line_sender_error& err) + { + std::cerr << "[ERROR] " << err.what() << std::endl; + return false; + } +} + +int main(int argc, const char* argv[]) +{ + auto host = "localhost"sv; + if (argc >= 2) + host = std::string_view{argv[1]}; + + auto port = "9009"sv; + if (argc >= 3) + port = std::string_view{argv[2]}; + + return !array_example(host, port); +} diff --git a/examples/line_sender_cpp_example_array_elem_strides.cpp b/examples/line_sender_cpp_example_array_elem_strides.cpp index 0065d009..a878d885 100644 --- a/examples/line_sender_cpp_example_array_elem_strides.cpp +++ b/examples/line_sender_cpp_example_array_elem_strides.cpp @@ -36,7 +36,8 @@ static bool array_example(std::string_view host, std::string_view port) questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_col, "BTC-USD"_utf8) - .column(book_col, 3, shape, strides, arr_data) + .column( + book_col, 3, shape, strides, arr_data) .at(questdb::ingress::timestamp_nanos::now()); sender.flush(buffer); return true; diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 7304c4a4..a46a20d5 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -491,6 +491,30 @@ bool line_sender_buffer_column_str( line_sender_utf8 value, line_sender_error** err_out); +/** + * Records a multidimensional array of 64-bit floating-point values in C-major + * order. + * + * @param[in] buffer Line buffer object. + * @param[in] name Column name. + * @param[in] rank Number of dimensions of the array. + * @param[in] shape Array of dimension sizes (length = `rank`). + * Each element must be a positive integer. + * @param[in] data_buffer First array element data. + * @param[in] data_buffer_len Bytes length of the array data. + * @param[out] err_out Set to an error object on failure (if non-NULL). + * @return true on success, false on error. + */ +LINESENDER_API +bool line_sender_buffer_column_f64_arr_c_major( + line_sender_buffer* buffer, + line_sender_column_name name, + size_t rank, + const uintptr_t* shape, + const uint8_t* data_buffer, + size_t data_buffer_len, + line_sender_error** err_out); + /** * Record a multidimensional array of double for the given column. * diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 91fc33d6..1711cf5e 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -109,6 +109,18 @@ enum class protocol_version v2 = 2, }; +enum class array_strides_mode +{ + /** Strides are inferred from C-style row-major memory layout. */ + c_major, + + /** Strides are provided in bytes */ + bytes, + + /** Strides are provided in elements */ + elems, +}; + /* Possible sources of the root certificates used to validate the server's TLS * certificate. */ enum class ca @@ -641,11 +653,9 @@ class line_sender_buffer } /** - * Record a multidimensional double-precision array for the given column. + * Records a multidimensional array of double-precision values. * - * @tparam B Strides mode selector: - * - `true` for byte-level strides - * - `false` for element-level strides + * @tparam Layout Memory layout specification (array_strides_mode) * @tparam T Element type (current only `double` is supported). * @tparam N Number of elements in the flat data array * @@ -654,7 +664,7 @@ class line_sender_buffer * @param data Array first element data. Size must match product of * dimensions. */ - template + template line_sender_buffer& column( column_name_view name, const size_t rank, @@ -666,16 +676,46 @@ class line_sender_buffer std::is_same_v, "Only double types are supported for arrays"); may_init(); - line_sender_error::wrapped_call( - B ? ::line_sender_buffer_column_f64_arr_byte_strides - : ::line_sender_buffer_column_f64_arr_elem_strides, - _impl, - name._impl, - rank, - shape.data(), - strides.data(), - reinterpret_cast(data.data()), - sizeof(double) * N); + switch (Layout) + { + case array_strides_mode::c_major: + if (!strides.empty()) + { + throw line_sender_error{ + line_sender_error_code::config_error, + "C_Major layout requires empty strides vector"}; + } + line_sender_error::wrapped_call( + ::line_sender_buffer_column_f64_arr_c_major, + _impl, + name._impl, + rank, + shape.data(), + reinterpret_cast(data.data()), + sizeof(double) * N); + break; + case array_strides_mode::bytes: + line_sender_error::wrapped_call( + ::line_sender_buffer_column_f64_arr_byte_strides, + _impl, + name._impl, + rank, + shape.data(), + strides.data(), + reinterpret_cast(data.data()), + sizeof(double) * N); + break; + case array_strides_mode::elems: + line_sender_error::wrapped_call( + ::line_sender_buffer_column_f64_arr_elem_strides, + _impl, + name._impl, + rank, + shape.data(), + strides.data(), + reinterpret_cast(data.data()), + sizeof(double) * N); + } return *this; } diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index eef6dc74..083b65e9 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -854,6 +854,46 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( true } +/// Records a float64 multidimensional array with **C-MAJOR memory layout**. +/// +/// @param[in] buffer Line buffer object. +/// @param[in] name Column name. +/// @param[in] rank Array dims. +/// @param[in] shape Array shape. +/// @param[in] data_buffer Array **first element** data memory ptr. +/// @param[in] data_buffer_len Array data memory length. +/// @param[out] err_out Set on error. +/// # Safety +/// - All pointer parameters must be valid and non-null +/// - shape must point to an array of `rank` integers +/// - data_buffer must point to a buffer of size `data_buffer_len` bytes +#[no_mangle] +pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_c_major( + buffer: *mut line_sender_buffer, + name: line_sender_column_name, + rank: size_t, + shape: *const usize, + data_buffer: *const u8, + data_buffer_len: size_t, + err_out: *mut *mut line_sender_error, +) -> bool { + let buffer = unwrap_buffer_mut(buffer); + let name = name.as_name(); + let view = match CMajorArrayView::::new(rank, shape, data_buffer, data_buffer_len) { + Ok(value) => value, + Err(err) => { + let err_ptr = Box::into_raw(Box::new(line_sender_error(err))); + *err_out = err_ptr; + return false; + } + }; + bubble_err_to_c!( + err_out, + buffer.column_arr::, CMajorArrayView<'_, f64>, f64>(name, &view) + ); + true +} + /// Records a float64 multidimensional array with **byte-level strides specification**. /// /// The `strides` represent byte offsets between elements along each dimension. @@ -1626,6 +1666,7 @@ pub unsafe extern "C" fn line_sender_now_micros() -> i64 { TimestampMicros::now().as_i64() } +use crate::ndarr::CMajorArrayView; #[cfg(feature = "confstr-ffi")] use questdb_confstr_ffi::questdb_conf_str_parse_err; diff --git a/questdb-rs-ffi/src/ndarr.rs b/questdb-rs-ffi/src/ndarr.rs index 0c7b12d6..e1d4d6ca 100644 --- a/questdb-rs-ffi/src/ndarr.rs +++ b/questdb-rs-ffi/src/ndarr.rs @@ -149,36 +149,7 @@ where data: *const u8, data_len: usize, ) -> Result { - if dims == 0 { - return Err(fmt_error!( - ArrayError, - "Zero-dimensional arrays are not supported", - )); - } - if data_len > MAX_ARRAY_BUFFER_SIZE { - return Err(fmt_error!( - ArrayError, - "Array buffer size too big: {}, maximum: {}", - data_len, - MAX_ARRAY_BUFFER_SIZE - )); - } - let shape = slice::from_raw_parts(shape, dims); - let size = shape - .iter() - .try_fold(std::mem::size_of::(), |acc, &dim| { - acc.checked_mul(dim) - .ok_or_else(|| fmt_error!(ArrayError, "Array buffer size too big")) - })?; - - if size != data_len { - return Err(fmt_error!( - ArrayError, - "Array buffer length mismatch (actual: {}, expected: {})", - data_len, - size - )); - } + let shape = check_array_shape::(dims, shape, data_len)?; let strides = slice::from_raw_parts(strides, dims); let mut slice = None; if data_len != 0 { @@ -261,6 +232,157 @@ where } } +#[derive(Debug)] +pub struct CMajorArrayView<'a, T> { + dims: usize, + shape: &'a [usize], + data: Option<&'a [u8]>, + _marker: std::marker::PhantomData, +} + +impl NdArrayView for CMajorArrayView<'_, T> +where + T: ArrayElement, +{ + type Iter<'b> + = CMajorArrayViewIterator<'b, T> + where + Self: 'b, + T: 'b; + + fn ndim(&self) -> usize { + self.dims + } + + fn dim(&self, index: usize) -> Result { + if index >= self.dims { + return Err(fmt_error!( + ArrayError, + "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", + index, + self.dims + )); + } + Ok(self.shape[index]) + } + + fn as_slice(&self) -> Option<&[T]> { + self.data.map(|d| unsafe { + slice::from_raw_parts(d.as_ptr() as *const T, d.len() / size_of::()) + }) + } + + fn iter(&self) -> Self::Iter<'_> { + let elem_size = self.shape.iter().product(); + let count = 0; + let data_ptr = match self.data { + Some(data) => data.as_ptr() as *const T, + None => std::ptr::null_mut(), + }; + CMajorArrayViewIterator { + elem_size, + count, + data_ptr, + _marker: Default::default(), + } + } +} + +pub struct CMajorArrayViewIterator<'a, T> { + elem_size: usize, + count: usize, + data_ptr: *const T, + _marker: std::marker::PhantomData<&'a T>, +} + +impl<'a, T> Iterator for CMajorArrayViewIterator<'a, T> +where + T: ArrayElement, +{ + type Item = &'a T; + + fn next(&mut self) -> Option { + if self.data_ptr.is_null() || self.count >= self.elem_size { + None + } else { + unsafe { + let ptr = self.data_ptr.add(self.count); + self.count += 1; + Some(&*(ptr)) + } + } + } +} + +impl CMajorArrayView<'_, T> +where + T: ArrayElement, +{ + /// Creates a new C-Major memory layout array view from raw components (unsafe constructor). + /// + /// # Safety + /// Caller must ensure all the following conditions: + /// - `shape` points to a valid array of at least `dims` elements + /// - `data` points to a valid memory block of at least `data_len` bytes + pub unsafe fn new( + dims: usize, + shape: *const usize, + data: *const u8, + data_len: usize, + ) -> Result { + let shape = check_array_shape::(dims, shape, data_len)?; + let mut slice = None; + if data_len != 0 { + slice = Some(slice::from_raw_parts(data, data_len)); + } + Ok(Self { + dims, + shape, + data: slice, + _marker: std::marker::PhantomData::, + }) + } +} + +fn check_array_shape( + dims: usize, + shape: *const usize, + data_len: usize, +) -> Result<&'static [usize], Error> { + if dims == 0 { + return Err(fmt_error!( + ArrayError, + "Zero-dimensional arrays are not supported", + )); + } + if data_len > MAX_ARRAY_BUFFER_SIZE { + return Err(fmt_error!( + ArrayError, + "Array buffer size too big: {}, maximum: {}", + data_len, + MAX_ARRAY_BUFFER_SIZE + )); + } + let shape = unsafe { slice::from_raw_parts(shape, dims) }; + + let size = shape + .iter() + .try_fold(std::mem::size_of::(), |acc, &dim| { + acc.checked_mul(dim) + .ok_or_else(|| fmt_error!(ArrayError, "Array buffer size too big")) + })?; + + if size != data_len { + return Err(fmt_error!( + ArrayError, + "Array buffer length mismatch (actual: {}, expected: {})", + data_len, + size + )); + } + Ok(shape) +} + #[cfg(test)] mod tests { use super::*; @@ -768,4 +890,78 @@ mod tests { assert_eq!(buf, expected); Ok(()) } + + #[test] + fn test_c_major_array_basic() -> TestResult { + let test_data = [1.1, 2.2, 3.3, 4.4]; + let array_view: CMajorArrayView<'_, f64> = unsafe { + CMajorArrayView::new( + 2, + [2, 2].as_ptr(), + test_data.as_ptr() as *const u8, + test_data.len() * 8usize, + ) + }?; + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &array_view)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', b'=', 14u8, // ARRAY_BINARY_FORMAT_TYPE + 10u8, // ArrayColumnTypeTag::Double.into() + 2u8 + ] + ); + assert_eq!( + &data[24..32], + [2i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[32..64], + &[ + 1.1f64.to_ne_bytes(), + 2.2f64.to_le_bytes(), + 3.3f64.to_le_bytes(), + 4.4f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) + } + + #[test] + fn test_c_major_empty_array() -> TestResult { + let test_data = []; + let array_view: CMajorArrayView<'_, f64> = unsafe { + CMajorArrayView::new( + 2, + [2, 0].as_ptr(), + test_data.as_ptr(), + test_data.len() * 8usize, + ) + }?; + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &array_view)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', b'=', 14u8, // ARRAY_BINARY_FORMAT_TYPE + 10u8, // ArrayColumnTypeTag::Double.into() + 2u8 + ] + ); + assert_eq!( + &data[24..32], + [2i32.to_le_bytes(), 0i32.to_le_bytes()].concat() + ); + Ok(()) + } } diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 0bc73da9..d6c997e3 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -300,6 +300,16 @@ def set_sig(fn, restype, *argtypes): c_uint8_p, c_size_t, c_line_sender_error_p_p) + set_sig( + dll.line_sender_buffer_column_f64_arr_c_major, + c_bool, + c_line_sender_buffer_p, + c_line_sender_column_name, + c_size_t, + c_size_t_p, + c_uint8_p, + c_size_t, + c_line_sender_error_p_p) set_sig( dll.line_sender_buffer_column_ts_nanos, c_bool, @@ -740,6 +750,31 @@ def _convert_tuple(tpl: tuple[int, ...], c_type: type, name: str) -> ctypes.POIN c_size_t(length) ) + def column_f64_arr_c_major(self, name: str, + rank: int, + shape: tuple[int, ...], + data: c_void_p, + length: int): + def _convert_tuple(tpl: tuple[int, ...], c_type: type, name: str) -> ctypes.POINTER: + arr_type = c_type * len(tpl) + try: + return arr_type(*[c_type(v) for v in tpl]) + except OverflowError as e: + raise ValueError( + f"{name} value exceeds {c_type.__name__} range" + ) from e + + c_shape = _convert_tuple(shape, c_size_t, "shape") + _error_wrapped_call( + _DLL.line_sender_buffer_column_f64_arr_c_major, + self._impl, + _column_name(name), + c_size_t(rank), + c_shape, + ctypes.cast(data, c_uint8_p), + c_size_t(length) + ) + def at_now(self): _error_wrapped_call( _DLL.line_sender_buffer_at_now, @@ -875,7 +910,10 @@ def column_f64_arr( array: numpy.ndarray): if array.dtype != numpy.float64: raise ValueError('expect float64 array') - self._buffer.column_f64_arr(name, array.ndim, array.shape, array.strides, array.ctypes.data, array.nbytes) + if array.flags.c_contiguous: + self._buffer.column_f64_arr_c_major(name, array.ndim, array.shape, array.ctypes.data, array.nbytes) + else: + self._buffer.column_f64_arr(name, array.ndim, array.shape, array.strides, array.ctypes.data, array.nbytes) return self def at_now(self): diff --git a/system_test/test.py b/system_test/test.py index 929dd766..f39fab36 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -785,6 +785,9 @@ def test_cpp_array_example(self): self._test_array_example( 'line_sender_cpp_example_array_elem_strides', 'cpp_market_orders_elem_strides', ) + self._test_array_example( + 'line_sender_cpp_example_array_c_major', + 'cpp_market_orders_c_major', ) def test_c_array_example(self): self._test_array_example( @@ -793,6 +796,9 @@ def test_c_array_example(self): self._test_array_example( 'line_sender_c_example_array_elem_strides', 'market_orders_elem_strides', ) + self._test_array_example( + 'line_sender_c_example_array_c_major', + 'market_orders_c_major', ) def _test_array_example(self, bin_name, table_name): if self.expected_protocol_version < qls.ProtocolVersion.V2: From 56d171bb80a0f5cacf22ebaa20a758bd4504dd6f Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 4 Jun 2025 23:51:20 +0800 Subject: [PATCH 09/46] optimize strideArrayView performance. --- questdb-rs-ffi/src/ndarr.rs | 83 ++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 48 deletions(-) diff --git a/questdb-rs-ffi/src/ndarr.rs b/questdb-rs-ffi/src/ndarr.rs index e1d4d6ca..340a3704 100644 --- a/questdb-rs-ffi/src/ndarr.rs +++ b/questdb-rs-ffi/src/ndarr.rs @@ -38,13 +38,6 @@ macro_rules! fmt_error { } /// A view into a multidimensional array with custom memory strides. -// TODO: We are currently evaluating whether to use StrideArrayView or ndarray's view. -// Current benchmarks show that StrideArrayView's iter implementation underperforms(2x) -// compared to ndarray's. -// We should optimise this implementation to be competitive. -// Unfortunately, the `ndarray` crate does not support negative strides -// which we need to support in this FFI crate for efficient iteration of -// numpy arrays coming from Python without copying the data. #[derive(Debug)] pub struct StrideArrayView<'a, T, const N: isize> { dims: usize, @@ -89,39 +82,24 @@ where } fn iter(&self) -> Self::Iter<'_> { - let mut dim_products = Vec::with_capacity(self.dims); - let mut product = 1; - for &dim in self.shape.iter().rev() { - dim_products.push(product); - product *= dim; - } - dim_products.reverse(); - // consider minus strides let base_ptr = match self.data { None => std::ptr::null(), - Some(data) => { - self.strides - .iter() - .enumerate() - .fold(data.as_ptr(), |ptr, (dim, &stride)| { - let stride_bytes_size = stride * N; - if stride_bytes_size < 0 { - let dim_size = self.shape[dim] as isize; - unsafe { ptr.offset(stride_bytes_size * (dim_size - 1)) } - } else { - ptr - } - }) - } + Some(data) => data.as_ptr(), }; + let mut cache_strides = Vec::with_capacity(self.dims); + for (&shape, &stride) in self.shape.iter().zip(self.strides) { + cache_strides.push((shape as isize - 1) * stride * N); + } RowMajorIter { base_ptr, array: self, - dim_products, + index: vec![0; self.dims], current_linear: 0, total_elements: self.shape.iter().product(), + next_offset: 0, + cache_strides, } } } @@ -195,9 +173,11 @@ where pub struct RowMajorIter<'a, T, const N: isize> { base_ptr: *const u8, array: &'a StrideArrayView<'a, T, N>, - dim_products: Vec, + index: Vec, current_linear: usize, total_elements: usize, + next_offset: isize, + cache_strides: Vec, } impl<'a, T, const N: isize> Iterator for RowMajorIter<'a, T, N> @@ -205,30 +185,37 @@ where T: ArrayElement, { type Item = &'a T; + fn next(&mut self) -> Option { if self.current_linear >= self.total_elements { return None; } - let mut remaining_index = self.current_linear; - let mut offset = 0; - - for (dim, &dim_factor) in self.dim_products.iter().enumerate() { - let coord = remaining_index / dim_factor; - remaining_index %= dim_factor; - let stride = self.array.strides[dim] * N; - let actual_coord = if stride >= 0 { - coord - } else { - self.array.shape[dim] - 1 - coord - }; - offset += actual_coord * stride.unsigned_abs(); - } - self.current_linear += 1; + let mut dim = self.array.dims - 1; + let offset = self.next_offset; + unsafe { - let ptr = self.base_ptr.add(offset); - Some(&*(ptr as *const T)) + loop { + let index = self.index.get_unchecked_mut(dim); + let shape = *self.array.shape.get_unchecked(dim); + let stride = *self.array.strides.get_unchecked(dim) * N; + if *index != shape - 1 { + *index += 1; + self.next_offset += stride; + break; + } else { + self.next_offset -= self.cache_strides.get_unchecked(dim); + *index = 0; + } + if dim == 0 { + break; + } + dim -= 1; + } } + + self.current_linear += 1; + unsafe { Some(&*(self.base_ptr.offset(offset) as *const T)) } } } From 680bdead3fea6002fffa1581e2647bd07aa22570 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 5 Jun 2025 09:08:19 +0800 Subject: [PATCH 10/46] add temp benchmark code. --- questdb-rs-ffi/Cargo.lock | 481 ++++++++++++++++++++++++++++++++ questdb-rs-ffi/Cargo.toml | 14 +- questdb-rs-ffi/benches/ndarr.rs | 54 ++++ questdb-rs-ffi/src/lib.rs | 2 +- 4 files changed, 548 insertions(+), 3 deletions(-) create mode 100644 questdb-rs-ffi/benches/ndarr.rs diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index d02045b8..529b178d 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -2,6 +2,33 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + [[package]] name = "base64" version = "0.22.1" @@ -20,12 +47,24 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +[[package]] +name = "bumpalo" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" + [[package]] name = "bytes" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.17" @@ -41,6 +80,58 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.5.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + [[package]] name = "core-foundation" version = "0.10.0" @@ -57,6 +148,73 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "is-terminal", + "itertools", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + [[package]] name = "dns-lookup" version = "2.0.4" @@ -69,6 +227,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "fnv" version = "1.0.7" @@ -98,6 +262,22 @@ dependencies = [ "wasi 0.14.2+wasi-0.2.4", ] +[[package]] +name = "half" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +dependencies = [ + "cfg-if", + "crunchy", +] + +[[package]] +name = "hermit-abi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" + [[package]] name = "http" version = "1.3.1" @@ -121,12 +301,42 @@ version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "libc" version = "0.2.171" @@ -139,18 +349,76 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "openssl-probe" version = "0.1.6" @@ -163,6 +431,49 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "portable-atomic" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -205,6 +516,7 @@ dependencies = [ "indoc", "itoa", "libc", + "ndarray", "questdb-confstr", "rand", "ring", @@ -226,7 +538,9 @@ dependencies = [ name = "questdb-rs-ffi" version = "5.0.0-rc1" dependencies = [ + "criterion", "libc", + "ndarray", "questdb-confstr-ffi", "questdb-rs", ] @@ -276,6 +590,61 @@ dependencies = [ "getrandom 0.3.2", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + [[package]] name = "ring" version = "0.17.14" @@ -343,12 +712,27 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + [[package]] name = "ryu" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.27" @@ -455,6 +839,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "unicode-ident" version = "1.0.18" @@ -508,6 +902,16 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -523,6 +927,74 @@ dependencies = [ "wit-bindgen-rt", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.26.8" @@ -548,6 +1020,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/questdb-rs-ffi/Cargo.toml b/questdb-rs-ffi/Cargo.toml index 1d461b5d..25084887 100644 --- a/questdb-rs-ffi/Cargo.toml +++ b/questdb-rs-ffi/Cargo.toml @@ -6,10 +6,11 @@ publish = false [lib] name = "questdb_client" -crate-type = ["cdylib", "staticlib"] +crate-type = ["cdylib", "staticlib", "lib"] [dependencies] libc = "0.2" +ndarray = { version = "0.16", optional = true } questdb-confstr-ffi = { version = "0.1.1", optional = true } [dependencies.questdb-rs] @@ -20,12 +21,21 @@ features = [ "insecure-skip-verify", "tls-webpki-certs", "tls-native-certs", - "ilp-over-http" + "ilp-over-http", + "ndarray" ] +[dev-dependencies] +criterion = "0.5" + [features] # Expose the config parsing C API. # This used by `py-questdb-client` to parse the config file. # It is exposed here to avoid having multiple copies of the `questdb-confstr` # crate in the final binary. confstr-ffi = ["dep:questdb-confstr-ffi"] + +[[bench]] +name = "ndarr" +harness = false +required-features = ["ndarray"] diff --git a/questdb-rs-ffi/benches/ndarr.rs b/questdb-rs-ffi/benches/ndarr.rs new file mode 100644 index 00000000..1e056834 --- /dev/null +++ b/questdb-rs-ffi/benches/ndarr.rs @@ -0,0 +1,54 @@ +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use ndarray::{Array, Array2}; +use questdb::ingress::{Buffer, ColumnName}; +use questdb_client::StrideArrayView; + +// benches NdArrayView and StridedArrayView write performance. +fn bench_array_view(c: &mut Criterion) { + let mut group = c.benchmark_group("write_array_view"); + let col_name = ColumnName::new("col1").unwrap(); + let array: Array2 = Array::ones((1000, 1000)); + let transposed_view = array.t(); + + // Case 1 + group.bench_function("ndarray_view", |b| { + let mut buffer = Buffer::new(questdb::ingress::ProtocolVersion::V2); + buffer.table("x1").unwrap(); + b.iter(|| { + buffer + .column_arr(col_name, black_box(&transposed_view)) + .unwrap(); + }); + buffer.clear(); + }); + + let elem_size = size_of::() as isize; + let strides: Vec = transposed_view + .strides() + .iter() + .map(|&s| s * elem_size) + .collect(); + let view2: StrideArrayView<'_, f64, 1> = unsafe { + StrideArrayView::new( + transposed_view.ndim(), + transposed_view.shape().as_ptr(), + strides.as_ptr(), + transposed_view.as_ptr() as *const u8, + transposed_view.len() * elem_size as usize, + ) + .unwrap() + }; + + // Case 2 + group.bench_function("strides_view", |b| { + let mut buffer = Buffer::new(questdb::ingress::ProtocolVersion::V2); + buffer.table("x1").unwrap(); + b.iter(|| { + buffer.column_arr(col_name, black_box(&view2)).unwrap(); + }); + buffer.clear(); + }); +} + +criterion_group!(benches, bench_array_view); +criterion_main!(benches); \ No newline at end of file diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 083b65e9..a0d16ab4 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -43,7 +43,7 @@ use questdb::{ }; mod ndarr; -use ndarr::StrideArrayView; +pub use ndarr::StrideArrayView; macro_rules! bubble_err_to_c { ($err_out:expr, $expression:expr) => { From dee8aead0f5ac380237c98bc0f25555bafbf7f44 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 5 Jun 2025 15:51:05 +0800 Subject: [PATCH 11/46] revert temp benchmark code. --- questdb-rs-ffi/Cargo.lock | 481 -------------------------------- questdb-rs-ffi/Cargo.toml | 14 +- questdb-rs-ffi/benches/ndarr.rs | 54 ---- questdb-rs-ffi/src/lib.rs | 2 +- 4 files changed, 3 insertions(+), 548 deletions(-) delete mode 100644 questdb-rs-ffi/benches/ndarr.rs diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index 529b178d..d02045b8 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -2,33 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - -[[package]] -name = "anstyle" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" - -[[package]] -name = "autocfg" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" - [[package]] name = "base64" version = "0.22.1" @@ -47,24 +20,12 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" -[[package]] -name = "bumpalo" -version = "3.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" - [[package]] name = "bytes" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - [[package]] name = "cc" version = "1.2.17" @@ -80,58 +41,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "ciborium" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - -[[package]] -name = "clap" -version = "4.5.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f" -dependencies = [ - "clap_builder", -] - -[[package]] -name = "clap_builder" -version = "4.5.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51" -dependencies = [ - "anstyle", - "clap_lex", -] - -[[package]] -name = "clap_lex" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" - [[package]] name = "core-foundation" version = "0.10.0" @@ -148,73 +57,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "criterion" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" -dependencies = [ - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "is-terminal", - "itertools", - "num-traits", - "once_cell", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast", - "itertools", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "crunchy" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" - [[package]] name = "dns-lookup" version = "2.0.4" @@ -227,12 +69,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - [[package]] name = "fnv" version = "1.0.7" @@ -262,22 +98,6 @@ dependencies = [ "wasi 0.14.2+wasi-0.2.4", ] -[[package]] -name = "half" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" -dependencies = [ - "cfg-if", - "crunchy", -] - -[[package]] -name = "hermit-abi" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" - [[package]] name = "http" version = "1.3.1" @@ -301,42 +121,12 @@ version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" -[[package]] -name = "is-terminal" -version = "0.4.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.59.0", -] - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" -[[package]] -name = "js-sys" -version = "0.3.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - [[package]] name = "libc" version = "0.2.171" @@ -349,76 +139,18 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" -[[package]] -name = "matrixmultiply" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" -dependencies = [ - "autocfg", - "rawpointer", -] - [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" -[[package]] -name = "ndarray" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" -dependencies = [ - "matrixmultiply", - "num-complex", - "num-integer", - "num-traits", - "portable-atomic", - "portable-atomic-util", - "rawpointer", -] - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - [[package]] name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" -[[package]] -name = "oorandom" -version = "11.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - [[package]] name = "openssl-probe" version = "0.1.6" @@ -431,49 +163,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - -[[package]] -name = "portable-atomic" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" - -[[package]] -name = "portable-atomic-util" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" -dependencies = [ - "portable-atomic", -] - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -516,7 +205,6 @@ dependencies = [ "indoc", "itoa", "libc", - "ndarray", "questdb-confstr", "rand", "ring", @@ -538,9 +226,7 @@ dependencies = [ name = "questdb-rs-ffi" version = "5.0.0-rc1" dependencies = [ - "criterion", "libc", - "ndarray", "questdb-confstr-ffi", "questdb-rs", ] @@ -590,61 +276,6 @@ dependencies = [ "getrandom 0.3.2", ] -[[package]] -name = "rawpointer" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" - -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "regex" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" - [[package]] name = "ring" version = "0.17.14" @@ -712,27 +343,12 @@ dependencies = [ "untrusted", ] -[[package]] -name = "rustversion" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" - [[package]] name = "ryu" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "schannel" version = "0.1.27" @@ -839,16 +455,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "unicode-ident" version = "1.0.18" @@ -902,16 +508,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -927,74 +523,6 @@ dependencies = [ "wit-bindgen-rt", ] -[[package]] -name = "wasm-bindgen" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "web-sys" -version = "0.3.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "webpki-roots" version = "0.26.8" @@ -1020,15 +548,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -[[package]] -name = "winapi-util" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/questdb-rs-ffi/Cargo.toml b/questdb-rs-ffi/Cargo.toml index 25084887..1d461b5d 100644 --- a/questdb-rs-ffi/Cargo.toml +++ b/questdb-rs-ffi/Cargo.toml @@ -6,11 +6,10 @@ publish = false [lib] name = "questdb_client" -crate-type = ["cdylib", "staticlib", "lib"] +crate-type = ["cdylib", "staticlib"] [dependencies] libc = "0.2" -ndarray = { version = "0.16", optional = true } questdb-confstr-ffi = { version = "0.1.1", optional = true } [dependencies.questdb-rs] @@ -21,21 +20,12 @@ features = [ "insecure-skip-verify", "tls-webpki-certs", "tls-native-certs", - "ilp-over-http", - "ndarray" + "ilp-over-http" ] -[dev-dependencies] -criterion = "0.5" - [features] # Expose the config parsing C API. # This used by `py-questdb-client` to parse the config file. # It is exposed here to avoid having multiple copies of the `questdb-confstr` # crate in the final binary. confstr-ffi = ["dep:questdb-confstr-ffi"] - -[[bench]] -name = "ndarr" -harness = false -required-features = ["ndarray"] diff --git a/questdb-rs-ffi/benches/ndarr.rs b/questdb-rs-ffi/benches/ndarr.rs deleted file mode 100644 index 1e056834..00000000 --- a/questdb-rs-ffi/benches/ndarr.rs +++ /dev/null @@ -1,54 +0,0 @@ -use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use ndarray::{Array, Array2}; -use questdb::ingress::{Buffer, ColumnName}; -use questdb_client::StrideArrayView; - -// benches NdArrayView and StridedArrayView write performance. -fn bench_array_view(c: &mut Criterion) { - let mut group = c.benchmark_group("write_array_view"); - let col_name = ColumnName::new("col1").unwrap(); - let array: Array2 = Array::ones((1000, 1000)); - let transposed_view = array.t(); - - // Case 1 - group.bench_function("ndarray_view", |b| { - let mut buffer = Buffer::new(questdb::ingress::ProtocolVersion::V2); - buffer.table("x1").unwrap(); - b.iter(|| { - buffer - .column_arr(col_name, black_box(&transposed_view)) - .unwrap(); - }); - buffer.clear(); - }); - - let elem_size = size_of::() as isize; - let strides: Vec = transposed_view - .strides() - .iter() - .map(|&s| s * elem_size) - .collect(); - let view2: StrideArrayView<'_, f64, 1> = unsafe { - StrideArrayView::new( - transposed_view.ndim(), - transposed_view.shape().as_ptr(), - strides.as_ptr(), - transposed_view.as_ptr() as *const u8, - transposed_view.len() * elem_size as usize, - ) - .unwrap() - }; - - // Case 2 - group.bench_function("strides_view", |b| { - let mut buffer = Buffer::new(questdb::ingress::ProtocolVersion::V2); - buffer.table("x1").unwrap(); - b.iter(|| { - buffer.column_arr(col_name, black_box(&view2)).unwrap(); - }); - buffer.clear(); - }); -} - -criterion_group!(benches, bench_array_view); -criterion_main!(benches); \ No newline at end of file diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index a0d16ab4..083b65e9 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -43,7 +43,7 @@ use questdb::{ }; mod ndarr; -pub use ndarr::StrideArrayView; +use ndarr::StrideArrayView; macro_rules! bubble_err_to_c { ($err_out:expr, $expression:expr) => { From 28a900f62718856b50277de02a88255e3d917489 Mon Sep 17 00:00:00 2001 From: victor Date: Thu, 5 Jun 2025 16:38:44 +0800 Subject: [PATCH 12/46] add flush_and_keep_with_flags --- cpp_test/test_line_sender.cpp | 24 +++++++++++-- include/questdb/ingress/line_sender.hpp | 48 ++++++++++++++++++++++++- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 9d80a8c9..4b808f8d 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -827,15 +827,35 @@ TEST_CASE("Test timestamp column.") const auto exp = ss.str(); CHECK(buffer.peek() == exp); - sender.flush_and_keep(buffer); + try + { + sender.flush_and_keep_with_flags(buffer, true); + CHECK_MESSAGE(false, "Expected exception"); + } + catch (const questdb::ingress::line_sender_error& se) + { + std::string msg{se.what()}; + CHECK_MESSAGE( + msg.rfind( + "Transactional flushes are not supported for ILP over TCP", + 0) == 0, + msg); + } + catch (...) + { + CHECK_MESSAGE(false, "Other exception raised."); + } + sender.flush_and_keep(buffer); + sender.flush_and_keep_with_flags(buffer, false); CHECK(buffer.peek() == exp); server.accept(); sender.close(); - CHECK(server.recv() == 1); + CHECK(server.recv() == 2); CHECK(server.msgs(0) == exp); + CHECK(server.msgs(1) == exp); } TEST_CASE("test timestamp_micros and timestamp_nanos::now()") diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 1711cf5e..fb09a878 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -1386,12 +1386,58 @@ class line_sender } } + /** + * Send the batch of rows in the buffer to the QuestDB server, and, if the + * parameter `transactional` is true, ensure the flush will be + * transactional. + * + * A flush is transactional iff all the rows belong to the same table. This + * allows QuestDB to treat the flush as a single database transaction, + * because it doesn't support transactions spanning multiple tables. + * Additionally, only ILP-over-HTTP supports transactional flushes. + * + * If the flush wouldn't be transactional, this function returns an error + * and doesn't flush any data. + * + * The function sends an HTTP request and waits for the response. If the + * server responds with an error, it returns a descriptive error. In the + * case of a network error, it retries until it has exhausted the retry time + * budget. + * + * All the data stays in the buffer. Clear the buffer before starting a new + * batch. + */ + void flush_and_keep_with_flags( + const line_sender_buffer& buffer, bool transactional) + { + if (buffer._impl) + { + ensure_impl(); + line_sender_error::wrapped_call( + ::line_sender_flush_and_keep_with_flags, + _impl, + buffer._impl, + transactional); + } + else + { + line_sender_buffer buffer2{this->protocol_version(), 0}; + buffer2.may_init(); + line_sender_error::wrapped_call( + ::line_sender_flush_and_keep_with_flags, + _impl, + buffer2._impl, + transactional); + } + } + /** * Check if an error occurred previously and the sender must be closed. * This happens when there was an earlier failure. * This method is specific to ILP-over-TCP and is not relevant for * ILP-over-HTTP. - * @return true if an error occurred with a sender and it must be closed. + * @return true if an error occurred with a sender and it must be + * closed. */ bool must_close() const noexcept { From 0a900e950c05887b486c5cba6fc5598f643ccca7 Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 6 Jun 2025 00:35:58 +0800 Subject: [PATCH 13/46] refactor `strideArrayView` to make iterator more efficient. --- questdb-rs-ffi/src/lib.rs | 125 ++++++++++++++++++++++------ questdb-rs-ffi/src/ndarr.rs | 162 ++++++++++++++++++------------------ 2 files changed, 180 insertions(+), 107 deletions(-) diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 083b65e9..b7a65139 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -61,6 +61,90 @@ macro_rules! bubble_err_to_c { }; } +#[macro_export] +macro_rules! fmt_error { + ($code:ident, $($arg:tt)*) => { + questdb::Error::new( + questdb::ErrorCode::$code, + format!($($arg)*)) + } +} + +macro_rules! new_stride_array { + ( + $rank:expr, + $m:literal, + $n:literal, + $shape:expr, + $strides:expr, + $data_buffer:expr, + $data_buffer_len:expr, + $err_out:expr, + $buffer:expr, + $name:expr + ) => {{ + let view = match StrideArrayView::::new( + $shape, + $strides, + $data_buffer, + $data_buffer_len, + ) { + Ok(value) => value, + Err(err) => { + let err_ptr = Box::into_raw(Box::new(line_sender_error(err))); + *$err_out = err_ptr; + return false; + } + }; + bubble_err_to_c!( + $err_out, + $buffer + .column_arr::, StrideArrayView<'_, f64, $m, $n>, f64>($name, &view) + ); + }}; +} + +macro_rules! generate_array_dims_branches { + ($rank:expr, $m:literal, $shape:expr, $strides:expr, $data_buffer:expr, $data_buffer_len:expr, $err_out:expr, $buffer:expr, $name:expr => $($n:literal),*) => { + match $rank { + 0 => { + let err = fmt_error!( + ArrayError, + "Zero-dimensional arrays are not supported", + ); + let err_ptr = Box::into_raw(Box::new(line_sender_error(err))); + *$err_out = err_ptr; + return false; + } + $( + $n => new_stride_array!( + $rank, + $m, + $n, + $shape, + $strides, + $data_buffer, + $data_buffer_len, + $err_out, + $buffer, + $name + ), + )* + other => { + let err = fmt_error!( + ArrayError, + "Array dimension mismatch: expected at most {} dimensions, but got {}", + 32, + other + ); + let err_ptr = Box::into_raw(Box::new(line_sender_error(err))); + *$err_out = err_ptr; + return false; + } + } + }; +} + /// Update the Rust builder inside the C opts object /// after calling a method that takes ownership of the builder. macro_rules! upd_opts { @@ -923,18 +1007,17 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_byte_strides( ) -> bool { let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); - let view = - match StrideArrayView::::new(rank, shape, strides, data_buffer, data_buffer_len) { - Ok(value) => value, - Err(err) => { - let err_ptr = Box::into_raw(Box::new(line_sender_error(err))); - *err_out = err_ptr; - return false; - } - }; - bubble_err_to_c!( + generate_array_dims_branches!( + rank, + 1, + shape, + strides, + data_buffer, + data_buffer_len, err_out, - buffer.column_arr::, StrideArrayView<'_, f64, 1>, f64>(name, &view) + buffer, + name + => 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32 ); true } @@ -969,24 +1052,18 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_elem_strides( ) -> bool { let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); - let view = match StrideArrayView::() as isize }>::new( + generate_array_dims_branches!( rank, + 8, shape, strides, data_buffer, data_buffer_len, - ) { - Ok(value) => value, - Err(err) => { - let err_ptr = Box::into_raw(Box::new(line_sender_error(err))); - *err_out = err_ptr; - return false; - } - }; - bubble_err_to_c!( - err_out, - buffer.column_arr::, StrideArrayView<'_, f64, { std::mem::size_of::() as isize }>, f64>(name, &view) - ); + err_out, + buffer, + name + => 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32 + ); true } diff --git a/questdb-rs-ffi/src/ndarr.rs b/questdb-rs-ffi/src/ndarr.rs index 340a3704..ed253fb5 100644 --- a/questdb-rs-ffi/src/ndarr.rs +++ b/questdb-rs-ffi/src/ndarr.rs @@ -22,6 +22,7 @@ * ******************************************************************************/ +use crate::fmt_error; use questdb::ingress::ArrayElement; use questdb::ingress::NdArrayView; use questdb::ingress::MAX_ARRAY_BUFFER_SIZE; @@ -29,45 +30,36 @@ use questdb::Error; use std::mem::size_of; use std::slice; -macro_rules! fmt_error { - ($code:ident, $($arg:tt)*) => { - questdb::Error::new( - questdb::ErrorCode::$code, - format!($($arg)*)) - } -} - /// A view into a multidimensional array with custom memory strides. #[derive(Debug)] -pub struct StrideArrayView<'a, T, const N: isize> { - dims: usize, +pub struct StrideArrayView<'a, T, const N: isize, const D: usize> { shape: &'a [usize], strides: &'a [isize], data: Option<&'a [u8]>, _marker: std::marker::PhantomData, } -impl NdArrayView for StrideArrayView<'_, T, N> +impl NdArrayView for StrideArrayView<'_, T, N, D> where T: ArrayElement, { type Iter<'b> - = RowMajorIter<'b, T, N> + = RowMajorIter<'b, T, N, D> where Self: 'b, T: 'b; fn ndim(&self) -> usize { - self.dims + D } fn dim(&self, index: usize) -> Result { - if index >= self.dims { + if index >= D { return Err(fmt_error!( ArrayError, "Dimension index out of bounds. Requested axis {}, but array only has {} dimension(s)", index, - self.dims + D )); } Ok(self.shape[index]) @@ -87,24 +79,17 @@ where None => std::ptr::null(), Some(data) => data.as_ptr(), }; - let mut cache_strides = Vec::with_capacity(self.dims); - for (&shape, &stride) in self.shape.iter().zip(self.strides) { - cache_strides.push((shape as isize - 1) * stride * N); - } - RowMajorIter { base_ptr, array: self, - index: vec![0; self.dims], + index: vec![0; D], current_linear: 0, total_elements: self.shape.iter().product(), - next_offset: 0, - cache_strides, } } } -impl StrideArrayView<'_, T, N> +impl StrideArrayView<'_, T, N, D> where T: ArrayElement, { @@ -121,20 +106,18 @@ where /// - Lifetime `'a` must outlive the view's usage /// - Strides are measured in bytes (not elements) pub unsafe fn new( - dims: usize, shape: *const usize, strides: *const isize, data: *const u8, data_len: usize, ) -> Result { - let shape = check_array_shape::(dims, shape, data_len)?; - let strides = slice::from_raw_parts(strides, dims); + let shape = check_array_shape::(D, shape, data_len)?; + let strides = slice::from_raw_parts(strides, D); let mut slice = None; if data_len != 0 { slice = Some(slice::from_raw_parts(data, data_len)); } Ok(Self { - dims, shape, strides, data: slice, @@ -152,7 +135,7 @@ where } let elem_size = size_of::() as isize; - if self.dims == 1 { + if D == 1 { return self.strides[0] * N == elem_size || self.shape[0] == 1; } @@ -170,17 +153,15 @@ where } /// Iterator for traversing a stride array in row-major (C-style) order. -pub struct RowMajorIter<'a, T, const N: isize> { +pub struct RowMajorIter<'a, T, const N: isize, const D: usize> { base_ptr: *const u8, - array: &'a StrideArrayView<'a, T, N>, + array: &'a StrideArrayView<'a, T, N, D>, index: Vec, current_linear: usize, total_elements: usize, - next_offset: isize, - cache_strides: Vec, } -impl<'a, T, const N: isize> Iterator for RowMajorIter<'a, T, N> +impl<'a, T, const N: isize, const D: usize> Iterator for RowMajorIter<'a, T, N, D> where T: ArrayElement, { @@ -190,27 +171,57 @@ where if self.current_linear >= self.total_elements { return None; } + let offset = unsafe { + match D { + 1 => { + let stride = *self.array.strides.get_unchecked(0) * N; + stride * self.current_linear as isize + } + 2 => { + let stride1 = self.array.strides.get_unchecked(0) * N; + let stride2 = self.array.strides.get_unchecked(1) * N; + let index1 = *self.index.get_unchecked(0); + let index2 = *self.index.get_unchecked(1); + if index2 != self.array.shape.get_unchecked(1) - 1 { + self.index[1] = index2 + 1; + } else { + self.index[1] = 0; + self.index[0] = index1 + 1; + } + index1 as isize * stride1 + index2 as isize * stride2 + } + 3 => { + let stride1 = self.array.strides.get_unchecked(0) * N; + let stride2 = self.array.strides.get_unchecked(1) * N; + let stride3 = self.array.strides.get_unchecked(2) * N; + let index1 = *self.index.get_unchecked(0); + let index2 = *self.index.get_unchecked(1); + let index3 = *self.index.get_unchecked(2); + + index1 as isize * stride1 + + index2 as isize * stride2 + + index3 as isize * stride3 + } + other => { + let mut offset = 0; + for dim in 0..other { + offset += *self.array.strides.get_unchecked(dim) + * N + * *self.index.get_unchecked(dim) as isize + } + offset + } + } + }; - let mut dim = self.array.dims - 1; - let offset = self.next_offset; - - unsafe { - loop { - let index = self.index.get_unchecked_mut(dim); - let shape = *self.array.shape.get_unchecked(dim); - let stride = *self.array.strides.get_unchecked(dim) * N; - if *index != shape - 1 { - *index += 1; - self.next_offset += stride; - break; + if D > 2 { + for (&dim, ix) in self.array.shape.iter().zip(self.index.iter_mut()).rev() { + *ix += 1; + if *ix == dim { + *ix = 0; } else { - self.next_offset -= self.cache_strides.get_unchecked(dim); - *index = 0; - } - if dim == 0 { break; } - dim -= 1; } } @@ -457,9 +468,8 @@ mod tests { let elem_size = std::mem::size_of::() as isize; let test_data = [1.1, 2.2, 3.3, 4.4]; - let array_view: StrideArrayView<'_, f64, 1> = unsafe { + let array_view: StrideArrayView<'_, f64, 1, 2> = unsafe { StrideArrayView::new( - 2, [2, 2].as_ptr(), [2 * elem_size, elem_size].as_ptr(), test_data.as_ptr() as *const u8, @@ -502,9 +512,8 @@ mod tests { let elem_size = std::mem::size_of::() as isize; let test_data = [1.1, 2.2, 3.3, 4.4]; - let array_view: StrideArrayView<'_, f64, 8> = unsafe { + let array_view: StrideArrayView<'_, f64, 8, 2> = unsafe { StrideArrayView::new( - 2, [2, 2].as_ptr(), [2, 1].as_ptr(), test_data.as_ptr() as *const u8, @@ -545,8 +554,7 @@ mod tests { #[test] fn test_stride_array_size_overflow() -> TestResult { let result = unsafe { - StrideArrayView::::new( - 2, + StrideArrayView::::new( [u32::MAX as usize, u32::MAX as usize].as_ptr(), [8, 8].as_ptr(), ptr::null(), @@ -563,9 +571,8 @@ mod tests { fn test_stride_view_length_mismatch() -> TestResult { let elem_size = size_of::() as isize; let under_data = [1.1]; - let result: Result, Error> = unsafe { + let result: Result, Error> = unsafe { StrideArrayView::new( - 2, [1, 2].as_ptr(), [elem_size, elem_size].as_ptr(), under_data.as_ptr() as *const u8, @@ -579,9 +586,8 @@ mod tests { .contains("Array buffer length mismatch (actual: 8, expected: 16)")); let over_data = [1.1, 2.2, 3.3]; - let result: Result, Error> = unsafe { + let result: Result, Error> = unsafe { StrideArrayView::new( - 2, [1, 2].as_ptr(), [elem_size, elem_size].as_ptr(), over_data.as_ptr() as *const u8, @@ -601,9 +607,8 @@ mod tests { fn test_stride_view_length_mismatch_with_elem_strides() -> TestResult { let elem_size = size_of::() as isize; let under_data = [1.1]; - let result: Result, Error> = unsafe { + let result: Result, Error> = unsafe { StrideArrayView::new( - 2, [1, 2].as_ptr(), [1, 1].as_ptr(), under_data.as_ptr() as *const u8, @@ -617,9 +622,8 @@ mod tests { .contains("Array buffer length mismatch (actual: 8, expected: 16)")); let over_data = [1.1, 2.2, 3.3]; - let result: Result, Error> = unsafe { + let result: Result, Error> = unsafe { StrideArrayView::new( - 2, [1, 2].as_ptr(), [elem_size, elem_size].as_ptr(), over_data.as_ptr() as *const u8, @@ -642,9 +646,8 @@ mod tests { let shape = [3usize, 2]; let strides = [elem_size, shape[0] as isize * elem_size]; - let array_view: StrideArrayView<'_, f64, 1> = unsafe { + let array_view: StrideArrayView<'_, f64, 1, 2> = unsafe { StrideArrayView::new( - shape.len(), shape.as_ptr(), strides.as_ptr(), col_major_data.as_ptr() as *const u8, @@ -679,9 +682,8 @@ mod tests { let shape = [3usize, 2]; let strides = [1, shape[0] as isize]; - let array_view: StrideArrayView<'_, f64, 8> = unsafe { + let array_view: StrideArrayView<'_, f64, 8, 2> = unsafe { StrideArrayView::new( - shape.len(), shape.as_ptr(), strides.as_ptr(), col_major_data.as_ptr() as *const u8, @@ -714,8 +716,7 @@ mod tests { let elem_size = size_of::(); let data = [1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; let view = unsafe { - StrideArrayView::::new( - 2, + StrideArrayView::::new( &[3usize, 3] as *const usize, &[-24isize, 8] as *const isize, (data.as_ptr() as *const u8).add(48), @@ -743,8 +744,7 @@ mod tests { let elem_size = size_of::(); let data = [1f64, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0]; let view = unsafe { - StrideArrayView::::new( - 2, + StrideArrayView::::new( &[3usize, 3] as *const usize, &[-3isize, 1] as *const isize, (data.as_ptr() as *const u8).add(48), @@ -771,17 +771,16 @@ mod tests { fn test_basic_edge_cases() -> TestResult { // empty array let elem_size = std::mem::size_of::() as isize; - let empty_view: StrideArrayView<'_, f64, 1> = - unsafe { StrideArrayView::new(2, [0, 0].as_ptr(), [0, 0].as_ptr(), ptr::null(), 0)? }; + let empty_view: StrideArrayView<'_, f64, 1, 2> = + unsafe { StrideArrayView::new([0, 0].as_ptr(), [0, 0].as_ptr(), ptr::null(), 0)? }; assert_eq!(empty_view.ndim(), 2); assert_eq!(empty_view.dim(0), Ok(0)); assert_eq!(empty_view.dim(1), Ok(0)); // single element array let single_data = [42.0]; - let single_view: StrideArrayView<'_, f64, 1> = unsafe { + let single_view: StrideArrayView<'_, f64, 1, 1> = unsafe { StrideArrayView::new( - 1, [1].as_ptr(), [elem_size].as_ptr(), single_data.as_ptr() as *const u8, @@ -804,8 +803,7 @@ mod tests { size_of::() as isize, ]; let array = unsafe { - StrideArrayView::::new( - shape.len(), + StrideArrayView::::new( shape.as_ptr(), strides.as_ptr(), test_data.as_ptr() as *const u8, @@ -832,8 +830,7 @@ mod tests { let shape = [2usize, 3]; let strides = [shape[1] as isize, 1]; let array = unsafe { - StrideArrayView::() as isize }>::new( - shape.len(), + StrideArrayView::() as isize }, 2>::new( shape.as_ptr(), strides.as_ptr(), test_data.as_ptr() as *const u8, @@ -861,8 +858,7 @@ mod tests { let shape = [2usize, 2]; let strides = [-8, -2]; let array = unsafe { - StrideArrayView::() as isize }>::new( - shape.len(), + StrideArrayView::() as isize }, 2>::new( shape.as_ptr(), strides.as_ptr(), test_data.as_ptr().add(11) as *const u8, From 1a8df97e377d026d773461bf21778ab23d40cce4 Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 6 Jun 2025 09:15:00 +0800 Subject: [PATCH 14/46] add 3d and 4d iterator tests. --- questdb-rs-ffi/src/ndarr.rs | 280 ++++++++++++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) diff --git a/questdb-rs-ffi/src/ndarr.rs b/questdb-rs-ffi/src/ndarr.rs index ed253fb5..71c9e689 100644 --- a/questdb-rs-ffi/src/ndarr.rs +++ b/questdb-rs-ffi/src/ndarr.rs @@ -947,4 +947,284 @@ mod tests { ); Ok(()) } + + #[test] + fn test_stride_non_contiguous_3d() -> TestResult { + let elem_size = size_of::() as isize; + let col_major_data = [ + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, + ]; + let shape = [2, 3, 2]; + let strides = [ + elem_size, + shape[0] as isize * elem_size, + shape[0] as isize * shape[1] as isize * elem_size, + ]; + let array_view: StrideArrayView<'_, f64, 1, 3> = unsafe { + StrideArrayView::new( + shape.as_ptr(), + strides.as_ptr(), + col_major_data.as_ptr() as *const u8, + col_major_data.len() * elem_size as usize, + ) + }?; + assert_eq!(array_view.ndim(), 3); + assert_eq!(array_view.dim(0), Ok(2)); + assert_eq!(array_view.dim(1), Ok(3)); + assert_eq!(array_view.dim(2), Ok(2)); + assert!(array_view.dim(3).is_err()); + assert!(array_view.as_slice().is_none()); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &array_view)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', b'=', 14u8, // ARRAY_BINARY_FORMAT_TYPE + 10u8, // ArrayColumnTypeTag::Double.into() + 3u8 + ] + ); + assert_eq!( + &data[24..36], + [2i32.to_le_bytes(), 3i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[36..132], + &[ + 1.0f64.to_ne_bytes(), + 7.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 9.0f64.to_le_bytes(), + 5.0f64.to_ne_bytes(), + 11.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 8.0f64.to_le_bytes(), + 4.0f64.to_ne_bytes(), + 10.0f64.to_le_bytes(), + 6.0f64.to_le_bytes(), + 12.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) + } + + #[test] + fn test_stride_non_contiguous_elem_stride_3d() -> TestResult { + let elem_size = size_of::() as isize; + let col_major_data = [ + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, + ]; + let shape = [2, 3, 2]; + let strides = [1, shape[0] as isize, shape[0] as isize * shape[1] as isize]; + let array_view: StrideArrayView<'_, f64, 8, 3> = unsafe { + StrideArrayView::new( + shape.as_ptr(), + strides.as_ptr(), + col_major_data.as_ptr() as *const u8, + col_major_data.len() * elem_size as usize, + ) + }?; + assert_eq!(array_view.ndim(), 3); + assert_eq!(array_view.dim(0), Ok(2)); + assert_eq!(array_view.dim(1), Ok(3)); + assert_eq!(array_view.dim(2), Ok(2)); + assert!(array_view.dim(3).is_err()); + assert!(array_view.as_slice().is_none()); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &array_view)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', b'=', 14u8, // ARRAY_BINARY_FORMAT_TYPE + 10u8, // ArrayColumnTypeTag::Double.into() + 3u8 + ] + ); + assert_eq!( + &data[24..36], + [2i32.to_le_bytes(), 3i32.to_le_bytes(), 2i32.to_le_bytes()].concat() + ); + assert_eq!( + &data[36..132], + &[ + 1.0f64.to_ne_bytes(), + 7.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 9.0f64.to_le_bytes(), + 5.0f64.to_ne_bytes(), + 11.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 8.0f64.to_le_bytes(), + 4.0f64.to_ne_bytes(), + 10.0f64.to_le_bytes(), + 6.0f64.to_le_bytes(), + 12.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) + } + + #[test] + fn test_stride_non_contiguous_4d() -> TestResult { + let elem_size = size_of::() as isize; + let col_major_data = [ + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, + ]; + let shape = [2, 2, 1, 3]; + let strides = [ + elem_size, + shape[0] as isize * elem_size, + shape[0] as isize * shape[1] as isize * elem_size, + shape[0] as isize * shape[1] as isize * shape[2] as isize * elem_size, + ]; + + let array_view: StrideArrayView<'_, f64, 1, 4> = unsafe { + StrideArrayView::new( + shape.as_ptr(), + strides.as_ptr(), + col_major_data.as_ptr() as *const u8, + col_major_data.len() * elem_size as usize, + ) + }?; + + assert_eq!(array_view.ndim(), 4); + assert_eq!(array_view.dim(0), Ok(2)); + assert_eq!(array_view.dim(1), Ok(2)); + assert_eq!(array_view.dim(2), Ok(1)); + assert_eq!(array_view.dim(3), Ok(3)); + assert!(array_view.dim(4).is_err()); + assert!(array_view.as_slice().is_none()); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &array_view)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', b'=', 14u8, // ARRAY_BINARY_FORMAT_TYPE + 10u8, // ArrayColumnTypeTag::Double.into() + 4u8 + ] + ); + assert_eq!( + &data[24..40], + [ + 2i32.to_le_bytes(), + 2i32.to_le_bytes(), + 1i32.to_le_bytes(), + 3i32.to_le_bytes() + ] + .concat() + ); + assert_eq!( + &data[40..136], + &[ + 1.0f64.to_ne_bytes(), + 5.0f64.to_le_bytes(), + 9.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 7.0f64.to_ne_bytes(), + 11.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 6.0f64.to_le_bytes(), + 10.0f64.to_ne_bytes(), + 4.0f64.to_le_bytes(), + 8.0f64.to_le_bytes(), + 12.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) + } + + #[test] + fn test_stride_non_contiguous_elem_stride_4d() -> TestResult { + let elem_size = size_of::() as isize; + let col_major_data = [ + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, + ]; + let shape = [2, 2, 1, 3]; + let strides = [ + 1, + shape[0] as isize, + shape[0] as isize * shape[1] as isize, + shape[0] as isize * shape[1] as isize * shape[2] as isize, + ]; + + let array_view: StrideArrayView<'_, f64, 8, 4> = unsafe { + StrideArrayView::new( + shape.as_ptr(), + strides.as_ptr(), + col_major_data.as_ptr() as *const u8, + col_major_data.len() * elem_size as usize, + ) + }?; + + assert_eq!(array_view.ndim(), 4); + assert_eq!(array_view.dim(0), Ok(2)); + assert_eq!(array_view.dim(1), Ok(2)); + assert_eq!(array_view.dim(2), Ok(1)); + assert_eq!(array_view.dim(3), Ok(3)); + assert!(array_view.dim(4).is_err()); + assert!(array_view.as_slice().is_none()); + + let mut buffer = Buffer::new(ProtocolVersion::V2); + buffer.table("my_test")?; + buffer.column_arr("temperature", &array_view)?; + let data = buffer.as_bytes(); + assert_eq!(&data[0..7], b"my_test"); + assert_eq!(&data[8..19], b"temperature"); + assert_eq!( + &data[19..24], + &[ + b'=', b'=', 14u8, // ARRAY_BINARY_FORMAT_TYPE + 10u8, // ArrayColumnTypeTag::Double.into() + 4u8 + ] + ); + assert_eq!( + &data[24..40], + [ + 2i32.to_le_bytes(), + 2i32.to_le_bytes(), + 1i32.to_le_bytes(), + 3i32.to_le_bytes() + ] + .concat() + ); + assert_eq!( + &data[40..136], + &[ + 1.0f64.to_ne_bytes(), + 5.0f64.to_le_bytes(), + 9.0f64.to_le_bytes(), + 3.0f64.to_le_bytes(), + 7.0f64.to_ne_bytes(), + 11.0f64.to_le_bytes(), + 2.0f64.to_le_bytes(), + 6.0f64.to_le_bytes(), + 10.0f64.to_ne_bytes(), + 4.0f64.to_le_bytes(), + 8.0f64.to_le_bytes(), + 12.0f64.to_le_bytes(), + ] + .concat() + ); + Ok(()) + } } From 94db7038fcc70bf7358f07bcbc6cc9275fc9a222 Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 6 Jun 2025 15:50:18 +0800 Subject: [PATCH 15/46] fix protocol_version docs and remove warning. --- include/questdb/ingress/line_sender.hpp | 14 +++++++ questdb-rs-ffi/src/lib.rs | 19 +++++---- questdb-rs/src/ingress/mod.rs | 55 +++++++++++++------------ 3 files changed, 55 insertions(+), 33 deletions(-) diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index fb09a878..fd3f05ab 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -1195,6 +1195,17 @@ class opts return *this; } + /** + * Sets the ingestion protocol version. + * + * HTTP transport automatically negotiates the protocol version by + * default(unset, strong recommended). You can explicitlyconfigure the + * protocol version to avoid the slight latency cost at connection time. + * + * TCP transport does not negotiate the protocol version and uses + * `protocol_version::v1` by default. You must explicitly set + * `protocol_version::v2` in order to ingest arrays. + */ opts& protocol_version(protocol_version version) noexcept { const auto c_protocol_version = @@ -1311,6 +1322,9 @@ class line_sender return *this; } + /** + * Get the current protocol version used by the sender. + */ questdb::ingress::protocol_version protocol_version() const noexcept { ensure_impl(); diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index b7a65139..357afea9 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -1344,7 +1344,12 @@ pub unsafe extern "C" fn line_sender_opts_token_y( upd_opts!(opts, err_out, token_y, token_y.as_str()) } -/// set the line protocol version. +/// Sets the ingestion protocol version. +/// - HTTP transport automatically negotiates the protocol version by default(unset, **Strong Recommended**). +/// You can explicitly configure the protocol version to avoid the slight latency cost at connection time. +/// - TCP transport does not negotiate the protocol version and uses [`ProtocolVersion::V1`] by +/// default. You must explicitly set [`ProtocolVersion::V2`] in order to ingest +/// arrays. #[no_mangle] pub unsafe extern "C" fn line_sender_opts_protocol_version( opts: *mut line_sender_opts, @@ -1594,10 +1599,11 @@ unsafe fn unwrap_sender_mut<'a>(sender: *mut line_sender) -> &'a mut Sender { &mut (*sender).0 } -/// Return the sender's protocol version. -/// This is either the protocol version that was set explicitly, -/// or the one that was auto-detected during the connection process. -/// If connecting via TCP and not overridden, the value is V1. +/// Returns the sender's protocol version +/// +/// - Explicitly set version, or +/// - Auto-detected during HTTP transport, or +/// - [`ProtocolVersion::V1`] for TCP transport. #[no_mangle] pub unsafe extern "C" fn line_sender_get_protocol_version( sender: *const line_sender, @@ -1610,8 +1616,7 @@ pub unsafe extern "C" fn line_sender_get_max_name_len(sender: *const line_sender unwrap_sender(sender).max_name_len() } -/// Construct a `line_sender_buffer` with a `max_name_len` of `127` and sender's default protocol version -/// which is the same as the QuestDB server default. +/// Construct a [`line_sender_buffer`] using the sender's protocol settings. #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_new_for_sender( sender: *const line_sender, diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index db7a9c74..d91cadb7 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -75,7 +75,7 @@ pub const MAX_ARRAY_DIM_LEN: usize = 0x0FFF_FFFF; // 1 << 28 - 1 pub enum ProtocolVersion { /// Version 1 of Line Protocol. /// Full-text protocol. - /// When used over HTTP, this version is compatible with the InfluxDB database. + /// This version is compatible with the InfluxDB database. V1 = 1, /// Version 2 of InfluxDB Line Protocol. @@ -607,22 +607,24 @@ impl Buffer { /// - Uses the specified protocol version /// - Sets maximum name length to **127 characters** (QuestDB server default) /// - /// This is equivalent to [`Sender::new_buffer`] when using the sender's - /// protocol version. For custom name lengths, use [`with_max_name_len`](Self::with_max_name_len) - /// or [`Sender::new_buffer_with_max_name_len`]. + /// This is equivalent to [`Sender::new_buffer`] when using the [`Sender::protocol_version`] + /// and [`Sender::max_name_len`] is 127. + /// + /// For custom name lengths, use [`Self::with_max_name_len`] pub fn new(protocol_version: ProtocolVersion) -> Self { Self::with_max_name_len(protocol_version, MAX_NAME_LEN_DEFAULT) } /// Creates a new [`Buffer`] with a custom maximum name length. /// - /// - `max_name_len`: Maximum allowed length for table/column names, must match + /// - `max_name_len`: Maximum allowed length for table/column names, match /// your QuestDB server's `cairo.max.file.name.length` configuration /// - `protocol_version`: Protocol version to use /// - /// This is equivalent to [`Sender::new_buffer_with_max_name_len`] when using - /// the sender's protocol version. For the default name length (127), - /// use [`new`](Self::new) or [`Sender::new_buffer`]. + /// This is equivalent to [`Sender::new_buffer`] when using the [`Sender::protocol_version`] + /// and [`Sender::max_name_len`]. + /// + /// For the default max name length limit (32), use [`Self::new`]. pub fn with_max_name_len(protocol_version: ProtocolVersion, max_name_len: usize) -> Self { Self { output: Vec::new(), @@ -2155,7 +2157,12 @@ impl SenderBuilder { Ok(self) } - /// Set the line protocol version. + /// Sets the ingestion protocol version. + /// - HTTP transport automatically negotiates the protocol version by default(unset, **Strong Recommended**). + /// You can explicitly configure the protocol version to avoid the slight latency cost at connection time. + /// - TCP transport does not negotiate the protocol version and uses [`ProtocolVersion::V1`] by + /// default. You must explicitly set [`ProtocolVersion::V2`] in order to ingest + /// arrays. pub fn protocol_version(mut self, protocol_version: ProtocolVersion) -> Result { self.protocol_version .set_specified("protocol_version", Some(protocol_version))?; @@ -2444,24 +2451,24 @@ impl SenderBuilder { pub_key_y: token_y.to_string(), }))), (protocol, Some(_username), Some(_password), None, None, None) - if protocol.is_tcpx() => { + if protocol.is_tcpx() => { Err(error::fmt!(ConfigError, r##"The "basic_auth" setting can only be used with the ILP/HTTP protocol."##, )) } (protocol, None, None, Some(_token), None, None) - if protocol.is_tcpx() => { + if protocol.is_tcpx() => { Err(error::fmt!(ConfigError, "Token authentication only be used with the ILP/HTTP protocol.")) } (protocol, _username, None, _token, _token_x, _token_y) - if protocol.is_tcpx() => { + if protocol.is_tcpx() => { Err(error::fmt!(ConfigError, r##"Incomplete ECDSA authentication parameters. Specify either all or none of: "username", "token", "token_x", "token_y"."##, )) } #[cfg(feature = "ilp-over-http")] (protocol, Some(username), Some(password), None, None, None) - if protocol.is_httpx() => { + if protocol.is_httpx() => { Ok(Some(AuthParams::Basic(BasicAuthParams { username: username.to_string(), password: password.to_string(), @@ -2469,21 +2476,21 @@ impl SenderBuilder { } #[cfg(feature = "ilp-over-http")] (protocol, Some(_username), None, None, None, None) - if protocol.is_httpx() => { + if protocol.is_httpx() => { Err(error::fmt!(ConfigError, r##"Basic authentication parameter "username" is present, but "password" is missing."##, )) } #[cfg(feature = "ilp-over-http")] (protocol, None, Some(_password), None, None, None) - if protocol.is_httpx() => { + if protocol.is_httpx() => { Err(error::fmt!(ConfigError, r##"Basic authentication parameter "password" is present, but "username" is missing."##, )) } #[cfg(feature = "ilp-over-http")] (protocol, None, None, Some(token), None, None) - if protocol.is_httpx() => { + if protocol.is_httpx() => { Ok(Some(AuthParams::Token(TokenAuthParams { token: token.to_string(), }))) @@ -2501,7 +2508,7 @@ impl SenderBuilder { } #[cfg(feature = "ilp-over-http")] (protocol, _username, _password, _token, None, None) - if protocol.is_httpx() => { + if protocol.is_httpx() => { Err(error::fmt!(ConfigError, r##"Inconsistent HTTP authentication parameters. Specify either "username" and "password", or just "token"."##, )) @@ -2844,11 +2851,7 @@ impl Sender { SenderBuilder::from_env()?.build() } - /// Creates a new [`Buffer`] with default parameters. - /// - /// This initializes a buffer using the sender's protocol version and - /// the QuestDB server's default maximum name length of 127 characters. - /// For custom name lengths, use [`new_buffer_with_max_name_len`](Self::new_buffer_with_max_name_len) + /// Creates a new [`Buffer`] using the sender's protocol settings pub fn new_buffer(&self) -> Buffer { Buffer::with_max_name_len(self.protocol_version, self.max_name_len) } @@ -2995,10 +2998,10 @@ impl Sender { !self.connected } - /// Return the sender's protocol version. - /// This is either the protocol version that was set explicitly, - /// or the one that was auto-detected during the connection process. - /// If connecting via TCP and not overridden, the value is V1. + /// Returns the sender's protocol version + /// + /// - Explicitly set version, or + /// - Auto-detected for HTTP transport, or [`ProtocolVersion::V1`] for TCP transport. pub fn protocol_version(&self) -> ProtocolVersion { self.protocol_version } From 3c4209f05a41f29551ec8619cc6a9ba4e6a349f4 Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 6 Jun 2025 16:21:35 +0800 Subject: [PATCH 16/46] fix protocol_version docs and remove warning. --- include/questdb/ingress/line_sender.h | 36 ++++++++++++++++----------- questdb-rs/src/ingress/mod.rs | 2 +- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index a46a20d5..16d22a0d 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -304,23 +304,20 @@ line_sender_column_name line_sender_column_name_assert( typedef struct line_sender_buffer line_sender_buffer; /** - * Construct a `line_sender_buffer` with a `max_name_len` of `127`, which is - * the same as the QuestDB server default. - * You should prefer to use `line_sender_for_sender()` instead, which - * automatically creates a buffer of the same protocol version as the sender. - * This is useful as it can rely on the sender's ability to auto-detect the - * protocol version when communicating over HTTP. + * Construct a `line_sender_buffer` with explicitly set `protocol_version` and + * fixed 127-byte name length limit. + * Prefer `line_sender_buffer_new_for_sender` which uses the sender's + * configured protocol settings. */ LINESENDER_API line_sender_buffer* line_sender_buffer_new( line_sender_protocol_version version); /** - * Construct a `line_sender_buffer` with a custom maximum length for table - * and column names. This should match the `cairo.max.file.name.length` - * setting of the QuestDB server you're connecting to. If the server does - * not configure it, the default is `127`, and you can call - * `line_sender_buffer_new()` instead. + * Construct a `line_sender_buffer` with explicitly set `protocol_version` and + * a max name length limit. + * Prefer `line_sender_buffer_new_for_sender` which uses the sender's + * configured protocol and max name length limit settings. */ LINESENDER_API line_sender_buffer* line_sender_buffer_with_max_name_len( @@ -813,7 +810,15 @@ bool line_sender_opts_token_y( line_sender_error** err_out); /** - * set the line protocol version. + * Sets the ingestion protocol version. + * + * HTTP transport automatically negotiates the protocol version by + * default(unset strong recommended). You can explicitlyconfigure the + * protocol version to avoid the slight latency cost at connection time. + * + * TCP transport does not negotiate the protocol version and uses + * `line_sender_protocol_version_1` by default. You must explicitly set + * `line_sender_protocol_version_2` in order to ingest arrays. */ LINESENDER_API bool line_sender_opts_protocol_version( @@ -991,8 +996,9 @@ line_sender* line_sender_from_env(line_sender_error** err_out); /** * Return the sender's protocol version. * This is either the protocol version that was set explicitly, - * or the one that was auto-detected during the connection process. - * If connecting via TCP and not overridden, the value is V1. + * or the one that was auto-detected during the connection process(Only for + * HTTP). If connecting via TCP and not overridden, the value is + * `line_sender_protocol_version_1`. */ LINESENDER_API line_sender_protocol_version line_sender_get_protocol_version( @@ -1006,7 +1012,7 @@ size_t line_sender_get_max_name_len(const line_sender* sender); /** * Construct a `line_sender_buffer` with the sender's - * configured protocol version and other parameters. + * configured `protocol_version` and `max_name_len` settings. * This is equivalent to calling: * line_sender_buffer_new( * line_sender_get_protocol_version(sender), diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index d91cadb7..60cb8fe1 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -2158,7 +2158,7 @@ impl SenderBuilder { } /// Sets the ingestion protocol version. - /// - HTTP transport automatically negotiates the protocol version by default(unset, **Strong Recommended**). + /// - HTTP transport automatically negotiates the protocol version by default(unset, **Strong Recommended**). /// You can explicitly configure the protocol version to avoid the slight latency cost at connection time. /// - TCP transport does not negotiate the protocol version and uses [`ProtocolVersion::V1`] by /// default. You must explicitly set [`ProtocolVersion::V2`] in order to ingest From d98a255662746590f6529a926b73bc86771f4bab Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 6 Jun 2025 18:01:32 +0800 Subject: [PATCH 17/46] introduce low-level c++ api to ingest array. --- cpp_test/test_line_sender.cpp | 32 +++-- include/questdb/ingress/line_sender.hpp | 163 +++++++++++++++++++----- 2 files changed, 152 insertions(+), 43 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 4b808f8d..f4ad5b75 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -273,8 +273,8 @@ TEST_CASE("line_sender c++ api basics") questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); // 3D array of doubles size_t rank = 3; - std::vector shape{2, 3, 2}; - std::vector strides{48, 16, 8}; + size_t shape[] = {2, 3, 2}; + ssize_t strides[] = {48, 16, 8}; std::array arr_data = { 48123.5, 2.4, @@ -288,31 +288,35 @@ TEST_CASE("line_sender c++ api basics") 2.7, 48121.5, 4.3}; - std::vector elem_strides{6, 2, 1}; + ssize_t elem_strides[] = {6, 2, 1}; buffer.table("test") .symbol("t1", "v1") .symbol("t2", "") .column("f1", 0.5) - .column( + .column( "a1", rank, shape, strides, arr_data) - .column( + .column( "a2", rank, shape, elem_strides, arr_data) - .column( - "a3", rank, shape, {}, arr_data) + .column("a3", rank, shape, arr_data) + .column( + "a4", rank, shape, strides, arr_data.data(), arr_data.size()) + .column( + "a5", rank, shape, elem_strides, arr_data.data(), arr_data.size()) + .column("a6", rank, shape, arr_data.data(), arr_data.size()) .at(questdb::ingress::timestamp_nanos{10000000}); CHECK(server.recv() == 0); - CHECK(buffer.size() == 386); + CHECK(buffer.size() == 734); sender.flush(buffer); CHECK(server.recv() == 1); std::string expect{"test,t1=v1,t2= f1=="}; push_double_to_buffer(expect, 0.5).append(",a1=="); - push_double_arr_to_buffer(expect, arr_data, 3, shape.data()) - .append(",a2=="); - push_double_arr_to_buffer(expect, arr_data, 3, shape.data()) - .append(",a3=="); - push_double_arr_to_buffer(expect, arr_data, 3, shape.data()) - .append(" 10000000\n"); + push_double_arr_to_buffer(expect, arr_data, 3, shape).append(",a2=="); + push_double_arr_to_buffer(expect, arr_data, 3, shape).append(",a3=="); + push_double_arr_to_buffer(expect, arr_data, 3, shape).append(",a4=="); + push_double_arr_to_buffer(expect, arr_data, 3, shape).append(",a5=="); + push_double_arr_to_buffer(expect, arr_data, 3, shape).append(",a6=="); + push_double_arr_to_buffer(expect, arr_data, 3, shape).append(" 10000000\n"); CHECK(server.msgs(0) == expect); } diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index fd3f05ab..1415b57a 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -109,11 +109,8 @@ enum class protocol_version v2 = 2, }; -enum class array_strides_mode +enum class array_strides_size_mode { - /** Strides are inferred from C-style row-major memory layout. */ - c_major, - /** Strides are provided in bytes */ bytes, @@ -655,70 +652,178 @@ class line_sender_buffer /** * Records a multidimensional array of double-precision values. * - * @tparam Layout Memory layout specification (array_strides_mode) + * @tparam L Array stride size mode (bytes or elements). * @tparam T Element type (current only `double` is supported). * @tparam N Number of elements in the flat data array * * @param name Column name. * @param shape Array dimensions (e.g., [2,3] for a 2x3 matrix). - * @param data Array first element data. Size must match product of - * dimensions. + * @param data Array data. */ - template + template line_sender_buffer& column( column_name_view name, const size_t rank, - const std::vector& shape, - const std::vector& strides, + const size_t* shape, + const ssize_t* strides, const std::array& data) { static_assert( std::is_same_v, "Only double types are supported for arrays"); may_init(); - switch (Layout) + switch (L) { - case array_strides_mode::c_major: - if (!strides.empty()) - { - throw line_sender_error{ - line_sender_error_code::config_error, - "C_Major layout requires empty strides vector"}; - } + case array_strides_size_mode::bytes: line_sender_error::wrapped_call( - ::line_sender_buffer_column_f64_arr_c_major, + ::line_sender_buffer_column_f64_arr_byte_strides, _impl, name._impl, rank, - shape.data(), + shape, + strides, reinterpret_cast(data.data()), sizeof(double) * N); break; - case array_strides_mode::bytes: + case array_strides_size_mode::elems: line_sender_error::wrapped_call( - ::line_sender_buffer_column_f64_arr_byte_strides, + ::line_sender_buffer_column_f64_arr_elem_strides, _impl, name._impl, rank, - shape.data(), - strides.data(), + shape, + strides, reinterpret_cast(data.data()), sizeof(double) * N); + } + return *this; + } + + /** + * Records a multidimensional array of double-precision values with c_major + * layout. + * + * @tparam T Element type (current only `double` is supported). + * @tparam N Number of elements in the flat data array + * + * @param name Column name. + * @param rank Number of dimensions of the array. + * @param shape Array dimensions (e.g., [2,3] for a 2x3 matrix). + * @param data Array data. + */ + template + line_sender_buffer& column( + column_name_view name, + const size_t rank, + const size_t* shape, + const std::array& data) + { + static_assert( + std::is_same_v, + "Only double types are supported for arrays"); + may_init(); + line_sender_error::wrapped_call( + ::line_sender_buffer_column_f64_arr_c_major, + _impl, + name._impl, + rank, + shape, + reinterpret_cast(data.data()), + sizeof(double) * N); + return *this; + } + + /** + * Records a multidimensional array of double-precision values with + * configurable stride mode. + * + * @tparam L Array stride size mode (bytes or elements). + * @tparam T Element type (current only `double` is supported). + * + * @param name Column name. + * @param rank Number of dimensions of the array. + * @param shape Array dimensions. Example: [2,3] for 2x3 matrix. + * @param strides Array strides. Step between consecutive elements in each + * dimension. + * @param data Raw pointer to the start of the array data. + * @param elem_count Total element of the array. + */ + template + line_sender_buffer& column( + column_name_view name, + const size_t rank, + const size_t* shape, + const ssize_t* strides, + const T* data, + size_t elem_count) + { + static_assert( + std::is_same_v, + "Only double types are supported for arrays"); + may_init(); + switch (L) + { + case array_strides_size_mode::bytes: + line_sender_error::wrapped_call( + ::line_sender_buffer_column_f64_arr_byte_strides, + _impl, + name._impl, + rank, + shape, + strides, + reinterpret_cast(data), + elem_count * sizeof(T)); break; - case array_strides_mode::elems: + case array_strides_size_mode::elems: line_sender_error::wrapped_call( ::line_sender_buffer_column_f64_arr_elem_strides, _impl, name._impl, rank, - shape.data(), - strides.data(), - reinterpret_cast(data.data()), - sizeof(double) * N); + shape, + strides, + reinterpret_cast(data), + elem_count * sizeof(T)); } return *this; } + /** + * Records a multidimensional array of double-precision values in C-major + * (row-major) layout layout. + * + * @tparam T Element type (current only `double` is supported). + * + * @param name Column name. + * @param rank Number of dimensions in the array. + * @param shape Array dimensions. Example: [2,3] for 2x3 matrix. + * @param data Raw pointer to the first element of the flat data + * array. + * @param elem_count Total element of the array. + */ + template + line_sender_buffer& column( + column_name_view name, + const size_t rank, + const size_t* shape, + const T* data, + size_t elem_count) + { + static_assert( + std::is_same_v, + "Only double types are supported for arrays"); + may_init(); + line_sender_error::wrapped_call( + ::line_sender_buffer_column_f64_arr_c_major, + _impl, + name._impl, + rank, + shape, + reinterpret_cast(data), + elem_count * sizeof(T)); + return *this; + } + /** * Record a string value for the given column. * @param name Column name. From 892fe9d4954b7b759d23638567974aad38252cdb Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 6 Jun 2025 18:02:16 +0800 Subject: [PATCH 18/46] introduce low-level c++ api to ingest array. --- examples/line_sender_cpp_example_array_byte_strides.cpp | 4 ++-- examples/line_sender_cpp_example_array_c_major.cpp | 3 +-- examples/line_sender_cpp_example_array_elem_strides.cpp | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/examples/line_sender_cpp_example_array_byte_strides.cpp b/examples/line_sender_cpp_example_array_byte_strides.cpp index 58505fbc..71680756 100644 --- a/examples/line_sender_cpp_example_array_byte_strides.cpp +++ b/examples/line_sender_cpp_example_array_byte_strides.cpp @@ -36,8 +36,8 @@ static bool array_example(std::string_view host, std::string_view port) questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_col, "BTC-USD"_utf8) - .column( - book_col, 3, shape, strides, arr_data) + .column( + book_col, 3, shape.data(), strides.data(), arr_data) .at(questdb::ingress::timestamp_nanos::now()); sender.flush(buffer); return true; diff --git a/examples/line_sender_cpp_example_array_c_major.cpp b/examples/line_sender_cpp_example_array_c_major.cpp index c5c44438..1b448012 100644 --- a/examples/line_sender_cpp_example_array_c_major.cpp +++ b/examples/line_sender_cpp_example_array_c_major.cpp @@ -35,8 +35,7 @@ static bool array_example(std::string_view host, std::string_view port) questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_col, "BTC-USD"_utf8) - .column( - book_col, 3, shape, {}, arr_data) + .column(book_col, 3, shape.data(), arr_data) .at(questdb::ingress::timestamp_nanos::now()); sender.flush(buffer); return true; diff --git a/examples/line_sender_cpp_example_array_elem_strides.cpp b/examples/line_sender_cpp_example_array_elem_strides.cpp index a878d885..ef64779d 100644 --- a/examples/line_sender_cpp_example_array_elem_strides.cpp +++ b/examples/line_sender_cpp_example_array_elem_strides.cpp @@ -36,8 +36,8 @@ static bool array_example(std::string_view host, std::string_view port) questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_col, "BTC-USD"_utf8) - .column( - book_col, 3, shape, strides, arr_data) + .column( + book_col, 3, shape.data(), strides.data(), arr_data) .at(questdb::ingress::timestamp_nanos::now()); sender.flush(buffer); return true; From 8fba18733b50b3120f6387103b91b1c836b496fd Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 6 Jun 2025 18:40:58 +0800 Subject: [PATCH 19/46] add array support docs. --- .../line_sender_c_example_array_byte_strides.c | 3 +++ examples/line_sender_c_example_array_c_major.c | 3 +++ .../line_sender_c_example_array_elem_strides.c | 3 +++ ...e_sender_cpp_example_array_byte_strides.cpp | 3 +++ .../line_sender_cpp_example_array_c_major.cpp | 3 +++ ...e_sender_cpp_example_array_elem_strides.cpp | 3 +++ include/questdb/ingress/line_sender.h | 11 +++++++++++ include/questdb/ingress/line_sender.hpp | 17 ++++++++++++++++- questdb-rs-ffi/src/lib.rs | 6 ++++++ questdb-rs/README.md | 2 ++ questdb-rs/examples/basic.rs | 1 + questdb-rs/examples/http.rs | 1 + questdb-rs/examples/protocol_version.rs | 3 ++- questdb-rs/src/ingress/mod.md | 18 +++++++++++++++++- questdb-rs/src/ingress/mod.rs | 5 +++++ 15 files changed, 79 insertions(+), 3 deletions(-) diff --git a/examples/line_sender_c_example_array_byte_strides.c b/examples/line_sender_c_example_array_byte_strides.c index 82e076e9..aa89987d 100644 --- a/examples/line_sender_c_example_array_byte_strides.c +++ b/examples/line_sender_c_example_array_byte_strides.c @@ -4,6 +4,9 @@ #include #include "concat.h" +/* + * QuestDB server version 8.4.0 or later is required for array support. + */ static bool example(const char* host, const char* port) { line_sender_error* err = NULL; diff --git a/examples/line_sender_c_example_array_c_major.c b/examples/line_sender_c_example_array_c_major.c index 806aa140..2e8b31e5 100644 --- a/examples/line_sender_c_example_array_c_major.c +++ b/examples/line_sender_c_example_array_c_major.c @@ -4,6 +4,9 @@ #include #include "concat.h" +/* + * QuestDB server version 8.4.0 or later is required for array support. + */ static bool example(const char* host, const char* port) { line_sender_error* err = NULL; diff --git a/examples/line_sender_c_example_array_elem_strides.c b/examples/line_sender_c_example_array_elem_strides.c index 44c81e05..bcfa1a8b 100644 --- a/examples/line_sender_c_example_array_elem_strides.c +++ b/examples/line_sender_c_example_array_elem_strides.c @@ -4,6 +4,9 @@ #include #include "concat.h" +/* + * QuestDB server version 8.4.0 or later is required for array support. + */ static bool example(const char* host, const char* port) { line_sender_error* err = NULL; diff --git a/examples/line_sender_cpp_example_array_byte_strides.cpp b/examples/line_sender_cpp_example_array_byte_strides.cpp index 71680756..0fc6f3ef 100644 --- a/examples/line_sender_cpp_example_array_byte_strides.cpp +++ b/examples/line_sender_cpp_example_array_byte_strides.cpp @@ -5,6 +5,9 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +/* + * QuestDB server version 8.4.0 or later is required for array support. + */ static bool array_example(std::string_view host, std::string_view port) { try diff --git a/examples/line_sender_cpp_example_array_c_major.cpp b/examples/line_sender_cpp_example_array_c_major.cpp index 1b448012..f74c379c 100644 --- a/examples/line_sender_cpp_example_array_c_major.cpp +++ b/examples/line_sender_cpp_example_array_c_major.cpp @@ -5,6 +5,9 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +/* + * QuestDB server version 8.4.0 or later is required for array support. + */ static bool array_example(std::string_view host, std::string_view port) { try diff --git a/examples/line_sender_cpp_example_array_elem_strides.cpp b/examples/line_sender_cpp_example_array_elem_strides.cpp index ef64779d..ea64e1b4 100644 --- a/examples/line_sender_cpp_example_array_elem_strides.cpp +++ b/examples/line_sender_cpp_example_array_elem_strides.cpp @@ -5,6 +5,9 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +/* + * QuestDB server version 8.4.0 or later is required for array support. + */ static bool array_example(std::string_view host, std::string_view port) { try diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 16d22a0d..f9b53cdd 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -115,6 +115,8 @@ typedef enum line_sender_protocol_version * Uses a binary format serialization for f64, and supports * the array data type. * This version is specific to QuestDB and not compatible with InfluxDB. + * QuestDB server version 8.4.0 or later is required for + * `line_sender_protocol_version_2` supported. */ line_sender_protocol_version_2 = 2, } line_sender_protocol_version; @@ -492,6 +494,8 @@ bool line_sender_buffer_column_str( * Records a multidimensional array of 64-bit floating-point values in C-major * order. * + * QuestDB server version 8.4.0 or later is required for array support. + * * @param[in] buffer Line buffer object. * @param[in] name Column name. * @param[in] rank Number of dimensions of the array. @@ -518,6 +522,8 @@ bool line_sender_buffer_column_f64_arr_c_major( * This API uses BYTE-LEVEL STRIDES where the stride values represent the * number of bytes between consecutive elements along each dimension. * + * QuestDB server version 8.4.0 or later is required for array support. + * * @param[in] buffer Line buffer object. * @param[in] name Column name. * @param[in] rank Number of dimensions of the array. @@ -546,6 +552,8 @@ bool line_sender_buffer_column_f64_arr_byte_strides( * This function uses ELEMENT-LEVEL STRIDES where the stride values represent * the number of elements between consecutive elements along each dimension. * + * QuestDB server version 8.4.0 or later is required for array support. + * * @param[in] buffer Line buffer object. * @param[in] name Column name. * @param[in] rank Number of dimensions of the array. @@ -819,6 +827,9 @@ bool line_sender_opts_token_y( * TCP transport does not negotiate the protocol version and uses * `line_sender_protocol_version_1` by default. You must explicitly set * `line_sender_protocol_version_2` in order to ingest arrays. + * + * QuestDB server version 8.4.0 or later is required for + * `line_sender_protocol_version_2`. */ LINESENDER_API bool line_sender_opts_protocol_version( diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 1415b57a..965388ff 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -105,7 +105,11 @@ enum class protocol_version /** InfluxDB Line Protocol v1. */ v1 = 1, - /** InfluxDB Line Protocol v2. */ + /** + * InfluxDB Line Protocol v2. + * QuestDB server version 8.4.0 or later is required for + * `v2` supported. + */ v2 = 2, }; @@ -652,6 +656,8 @@ class line_sender_buffer /** * Records a multidimensional array of double-precision values. * + * QuestDB server version 8.4.0 or later is required for array support. + * * @tparam L Array stride size mode (bytes or elements). * @tparam T Element type (current only `double` is supported). * @tparam N Number of elements in the flat data array @@ -703,6 +709,8 @@ class line_sender_buffer * Records a multidimensional array of double-precision values with c_major * layout. * + * QuestDB server version 8.4.0 or later is required for array support. + * * @tparam T Element type (current only `double` is supported). * @tparam N Number of elements in the flat data array * @@ -737,6 +745,8 @@ class line_sender_buffer * Records a multidimensional array of double-precision values with * configurable stride mode. * + * QuestDB server version 8.4.0 or later is required for array support. + * * @tparam L Array stride size mode (bytes or elements). * @tparam T Element type (current only `double` is supported). * @@ -792,6 +802,8 @@ class line_sender_buffer * Records a multidimensional array of double-precision values in C-major * (row-major) layout layout. * + * QuestDB server version 8.4.0 or later is required for array support. + * * @tparam T Element type (current only `double` is supported). * * @param name Column name. @@ -1310,6 +1322,9 @@ class opts * TCP transport does not negotiate the protocol version and uses * `protocol_version::v1` by default. You must explicitly set * `protocol_version::v2` in order to ingest arrays. + * + * QuestDB server version 8.4.0 or later is required for + * `protocol_version::v2`. */ opts& protocol_version(protocol_version version) noexcept { diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 357afea9..ff18cc0f 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -312,6 +312,7 @@ pub enum ProtocolVersion { /// Version 2 of InfluxDB Line Protocol. /// Uses binary format serialization for f64, and supports the array data type. /// This version is specific to QuestDB and is not compatible with InfluxDB. + /// QuestDB server version 8.4.0 or later is required for V2 supported. V2 = 2, } @@ -951,6 +952,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( /// - All pointer parameters must be valid and non-null /// - shape must point to an array of `rank` integers /// - data_buffer must point to a buffer of size `data_buffer_len` bytes +/// - QuestDB server version 8.4.0 or later is required for array support. #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_c_major( buffer: *mut line_sender_buffer, @@ -994,6 +996,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_c_major( /// - All pointer parameters must be valid and non-null /// - shape must point to an array of `rank` integers /// - data_buffer must point to a buffer of size `data_buffer_len` bytes +/// - QuestDB server version 8.4.0 or later is required for array support. #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_byte_strides( buffer: *mut line_sender_buffer, @@ -1039,6 +1042,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_byte_strides( /// - All pointer parameters must be valid and non-null /// - shape must point to an array of `rank` integers /// - data_buffer must point to a buffer of size `data_buffer_len` bytes +/// - QuestDB server version 8.4.0 or later is required for array support. #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_elem_strides( buffer: *mut line_sender_buffer, @@ -1350,6 +1354,8 @@ pub unsafe extern "C" fn line_sender_opts_token_y( /// - TCP transport does not negotiate the protocol version and uses [`ProtocolVersion::V1`] by /// default. You must explicitly set [`ProtocolVersion::V2`] in order to ingest /// arrays. +/// +/// QuestDB server version 8.4.0 or later is required for [`ProtocolVersion::V2`] #[no_mangle] pub unsafe extern "C" fn line_sender_opts_protocol_version( opts: *mut line_sender_opts, diff --git a/questdb-rs/README.md b/questdb-rs/README.md index 61eb85d3..8a6a315a 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -27,6 +27,8 @@ These protocol versions are supported over both HTTP and TCP. | **1** | Over HTTP it's compatible InfluxDB Line Protocol (ILP) | All QuestDB versions | | **2** | 64-bit floats sent as binary, adds n-dimentional arrays | 8.4.0+ (2023-10-30) | +**Note**: QuestDB server version 8.4.0 or later is required for `protocol_version=2`. + ## Quick Start To start using `questdb-rs`, add it as a dependency of your project: diff --git a/questdb-rs/examples/basic.rs b/questdb-rs/examples/basic.rs index 853003b9..3eca12b7 100644 --- a/questdb-rs/examples/basic.rs +++ b/questdb-rs/examples/basic.rs @@ -18,6 +18,7 @@ fn main() -> Result<()> { .symbol("side", "sell")? .column_f64("price", 2615.54)? .column_f64("amount", 0.00044)? + // QuestDB server version 8.4.0 or later is required for array support. .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? .at(designated_timestamp)?; diff --git a/questdb-rs/examples/http.rs b/questdb-rs/examples/http.rs index f9e5954e..a59a2b53 100644 --- a/questdb-rs/examples/http.rs +++ b/questdb-rs/examples/http.rs @@ -13,6 +13,7 @@ fn main() -> Result<()> { .symbol("side", "sell")? .column_f64("price", 2615.54)? .column_f64("amount", 0.00044)? + // QuestDB server version 8.4.0 or later is required for array support. .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? .at(TimestampNanos::now())?; sender.flush(&mut buffer)?; diff --git a/questdb-rs/examples/protocol_version.rs b/questdb-rs/examples/protocol_version.rs index 0d184d90..ece1366f 100644 --- a/questdb-rs/examples/protocol_version.rs +++ b/questdb-rs/examples/protocol_version.rs @@ -18,8 +18,9 @@ fn main() -> Result<()> { .at(TimestampNanos::now())?; sender.flush(&mut buffer)?; + // QuestDB server version 8.4.0 or later is required for protocol_version=2. let mut sender2 = Sender::from_conf( - "https::addr=localhost:9000;username=foo;password=bar;protocol_version=1;", + "https::addr=localhost:9000;username=foo;password=bar;protocol_version=2;", )?; let mut buffer2 = sender.new_buffer(); buffer2 diff --git a/questdb-rs/src/ingress/mod.md b/questdb-rs/src/ingress/mod.md index e5f947f0..4f1ec31d 100644 --- a/questdb-rs/src/ingress/mod.md +++ b/questdb-rs/src/ingress/mod.md @@ -254,7 +254,23 @@ However, TCP has a lower overhead than HTTP and it's worthwhile to try out as an alternative in a scenario where you have a constantly high data rate and/or deal with a high-latency network connection. -### Timestamp Column Name +## Array Datatype + +The [`Sender::column_arr`](Sender::column_arr) method supports efficient ingestion of N-dimensional +arrays using several convenient types: + +- native Rust arrays and slices (up to 3-dimensional) +- native Rust vectors (up to 3-dimensional) +- arrays from the [`ndarray`](https://docs.rs/ndarray) crate + +You must use protocol version 2 to ingest arrays. HTTP transport will +automatically enable it as long as you're connecting to an up-to-date QuestDB +server (version 8.4.0 or later), but with TCP you must explicitly specify it in +the configuration string: `protocol_version=2;`. + +**Note**: QuestDB server version 8.4.0 or later is required for array support. + +## Timestamp Column Name The InfluxDB Line Protocol (ILP) does not give a name to the designated timestamp, so if you let this client auto-create the table, it will have the default `timestamp` name. diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 60cb8fe1..5204abc6 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -81,6 +81,7 @@ pub enum ProtocolVersion { /// Version 2 of InfluxDB Line Protocol. /// Uses binary format serialization for f64, and supports the array data type. /// This version is specific to QuestDB and is not compatible with InfluxDB. + /// QuestDB server version 8.4.0 or later is required for V2 supported. V2 = 2, } @@ -1089,6 +1090,8 @@ impl Buffer { /// Supports arrays with up to [`MAX_ARRAY_DIMS`] dimensions. The array elements must /// be of type `f64`, which is currently the only supported data type. /// + /// **Note**: QuestDB server version 8.4.0 or later is required for array support. + /// /// # Examples /// /// Recording a 2D array using slices: @@ -2163,6 +2166,8 @@ impl SenderBuilder { /// - TCP transport does not negotiate the protocol version and uses [`ProtocolVersion::V1`] by /// default. You must explicitly set [`ProtocolVersion::V2`] in order to ingest /// arrays. + /// + /// **Note**: QuestDB server version 8.4.0 or later is required for protocol_version=2. pub fn protocol_version(mut self, protocol_version: ProtocolVersion) -> Result { self.protocol_version .set_specified("protocol_version", Some(protocol_version))?; From 515244552bd8db3c4bbb468db9676514f10f1d7b Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 6 Jun 2025 18:48:58 +0800 Subject: [PATCH 20/46] fix protocol_version docs and remove warning. --- include/questdb/ingress/line_sender.h | 4 ++-- include/questdb/ingress/line_sender.hpp | 4 ++-- questdb-rs-ffi/src/lib.rs | 4 ++-- questdb-rs/README.md | 2 +- questdb-rs/examples/protocol_version.rs | 2 +- questdb-rs/src/ingress/mod.md | 2 +- questdb-rs/src/ingress/mod.rs | 4 ++-- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index f9b53cdd..4c692820 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -116,7 +116,7 @@ typedef enum line_sender_protocol_version * the array data type. * This version is specific to QuestDB and not compatible with InfluxDB. * QuestDB server version 8.4.0 or later is required for - * `line_sender_protocol_version_2` supported. + * `line_sender_protocol_version_2` support. */ line_sender_protocol_version_2 = 2, } line_sender_protocol_version; @@ -829,7 +829,7 @@ bool line_sender_opts_token_y( * `line_sender_protocol_version_2` in order to ingest arrays. * * QuestDB server version 8.4.0 or later is required for - * `line_sender_protocol_version_2`. + * `line_sender_protocol_version_2` support. */ LINESENDER_API bool line_sender_opts_protocol_version( diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 965388ff..8a9215e7 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -108,7 +108,7 @@ enum class protocol_version /** * InfluxDB Line Protocol v2. * QuestDB server version 8.4.0 or later is required for - * `v2` supported. + * `v2` support. */ v2 = 2, }; @@ -1324,7 +1324,7 @@ class opts * `protocol_version::v2` in order to ingest arrays. * * QuestDB server version 8.4.0 or later is required for - * `protocol_version::v2`. + * `protocol_version::v2` support. */ opts& protocol_version(protocol_version version) noexcept { diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index ff18cc0f..baab0cba 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -312,7 +312,7 @@ pub enum ProtocolVersion { /// Version 2 of InfluxDB Line Protocol. /// Uses binary format serialization for f64, and supports the array data type. /// This version is specific to QuestDB and is not compatible with InfluxDB. - /// QuestDB server version 8.4.0 or later is required for V2 supported. + /// QuestDB server version 8.4.0 or later is required for `V2` supported. V2 = 2, } @@ -1355,7 +1355,7 @@ pub unsafe extern "C" fn line_sender_opts_token_y( /// default. You must explicitly set [`ProtocolVersion::V2`] in order to ingest /// arrays. /// -/// QuestDB server version 8.4.0 or later is required for [`ProtocolVersion::V2`] +/// QuestDB server version 8.4.0 or later is required for [`ProtocolVersion::V2`] support #[no_mangle] pub unsafe extern "C" fn line_sender_opts_protocol_version( opts: *mut line_sender_opts, diff --git a/questdb-rs/README.md b/questdb-rs/README.md index 8a6a315a..292fb463 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -27,7 +27,7 @@ These protocol versions are supported over both HTTP and TCP. | **1** | Over HTTP it's compatible InfluxDB Line Protocol (ILP) | All QuestDB versions | | **2** | 64-bit floats sent as binary, adds n-dimentional arrays | 8.4.0+ (2023-10-30) | -**Note**: QuestDB server version 8.4.0 or later is required for `protocol_version=2`. +**Note**: QuestDB server version 8.4.0 or later is required for `protocol_version=2` support. ## Quick Start diff --git a/questdb-rs/examples/protocol_version.rs b/questdb-rs/examples/protocol_version.rs index ece1366f..84b6ee8b 100644 --- a/questdb-rs/examples/protocol_version.rs +++ b/questdb-rs/examples/protocol_version.rs @@ -18,7 +18,7 @@ fn main() -> Result<()> { .at(TimestampNanos::now())?; sender.flush(&mut buffer)?; - // QuestDB server version 8.4.0 or later is required for protocol_version=2. + // QuestDB server version 8.4.0 or later is required for `protocol_version=2` support. let mut sender2 = Sender::from_conf( "https::addr=localhost:9000;username=foo;password=bar;protocol_version=2;", )?; diff --git a/questdb-rs/src/ingress/mod.md b/questdb-rs/src/ingress/mod.md index 4f1ec31d..2ba316b0 100644 --- a/questdb-rs/src/ingress/mod.md +++ b/questdb-rs/src/ingress/mod.md @@ -256,7 +256,7 @@ with a high-latency network connection. ## Array Datatype -The [`Sender::column_arr`](Sender::column_arr) method supports efficient ingestion of N-dimensional +The [`Buffer::column_arr`](Buffer::column_arr) method supports efficient ingestion of N-dimensional arrays using several convenient types: - native Rust arrays and slices (up to 3-dimensional) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 5204abc6..4b239a2a 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -81,7 +81,7 @@ pub enum ProtocolVersion { /// Version 2 of InfluxDB Line Protocol. /// Uses binary format serialization for f64, and supports the array data type. /// This version is specific to QuestDB and is not compatible with InfluxDB. - /// QuestDB server version 8.4.0 or later is required for V2 supported. + /// QuestDB server version 8.4.0 or later is required for `V2` supported. V2 = 2, } @@ -2167,7 +2167,7 @@ impl SenderBuilder { /// default. You must explicitly set [`ProtocolVersion::V2`] in order to ingest /// arrays. /// - /// **Note**: QuestDB server version 8.4.0 or later is required for protocol_version=2. + /// **Note**: QuestDB server version 8.4.0 or later is required for [`ProtocolVersion::V2`] support. pub fn protocol_version(mut self, protocol_version: ProtocolVersion) -> Result { self.protocol_version .set_specified("protocol_version", Some(protocol_version))?; From 48389d57e6dae3851556e1ed6e9da1f4da25f1c3 Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 6 Jun 2025 18:57:53 +0800 Subject: [PATCH 21/46] change ssize_t to intptr_t --- cpp_test/test_line_sender.cpp | 6 +++--- include/questdb/ingress/line_sender.hpp | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index f4ad5b75..d06d0e57 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -273,8 +273,8 @@ TEST_CASE("line_sender c++ api basics") questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); // 3D array of doubles size_t rank = 3; - size_t shape[] = {2, 3, 2}; - ssize_t strides[] = {48, 16, 8}; + uintptr_t shape[] = {2, 3, 2}; + intptr_t strides[] = {48, 16, 8}; std::array arr_data = { 48123.5, 2.4, @@ -288,7 +288,7 @@ TEST_CASE("line_sender c++ api basics") 2.7, 48121.5, 4.3}; - ssize_t elem_strides[] = {6, 2, 1}; + intptr_t elem_strides[] = {6, 2, 1}; buffer.table("test") .symbol("t1", "v1") .symbol("t2", "") diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 8a9215e7..22a3e6e0 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -670,8 +670,8 @@ class line_sender_buffer line_sender_buffer& column( column_name_view name, const size_t rank, - const size_t* shape, - const ssize_t* strides, + const uintptr_t* shape, + const intptr_t* strides, const std::array& data) { static_assert( @@ -762,8 +762,8 @@ class line_sender_buffer line_sender_buffer& column( column_name_view name, const size_t rank, - const size_t* shape, - const ssize_t* strides, + const uintptr_t* shape, + const intptr_t* strides, const T* data, size_t elem_count) { From 87e237cd18571210f010fe2afc5ac8016359daee Mon Sep 17 00:00:00 2001 From: victor Date: Fri, 6 Jun 2025 21:52:53 +0800 Subject: [PATCH 22/46] add high level c++ api --- cpp_test/test_line_sender.cpp | 111 ++++++++++++++++++++++++ include/questdb/ingress/line_sender.hpp | 62 +++++++++++++ 2 files changed, 173 insertions(+) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index d06d0e57..7898c739 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -111,6 +111,24 @@ std::string& push_double_arr_to_buffer( return buffer; } +std::string& push_double_arr_to_buffer( + std::string& buffer, + std::vector& data, + size_t rank, + uintptr_t* shape) +{ + buffer.push_back(14); + buffer.push_back(10); + buffer.push_back(static_cast(rank)); + for (size_t i = 0; i < rank; ++i) + buffer.append( + reinterpret_cast(&shape[i]), sizeof(uint32_t)); + buffer.append( + reinterpret_cast(data.data()), + data.size() * sizeof(double)); + return buffer; +} + std::string& push_double_to_buffer(std::string& buffer, double data) { buffer.push_back(16); @@ -320,6 +338,99 @@ TEST_CASE("line_sender c++ api basics") CHECK(server.msgs(0) == expect); } +TEST_CASE("line_sender array vector API") +{ + questdb::ingress::test::mock_server server; + questdb::ingress::opts opts{ + questdb::ingress::protocol::tcp, + std::string("127.0.0.1"), + std::to_string(server.port())}; + opts.protocol_version(questdb::ingress::protocol_version::v2); + questdb::ingress::line_sender sender{opts}; + CHECK_FALSE(sender.must_close()); + server.accept(); + CHECK(server.recv() == 0); + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + std::vector arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + buffer.table("test") + .symbol("t1", "v1") + .symbol("t2", "") + .column("a1", arr_data) + .at(questdb::ingress::timestamp_nanos{10000000}); + + uintptr_t test_shape[] = {12}; + CHECK(server.recv() == 0); + CHECK(buffer.size() == 132); + sender.flush(buffer); + CHECK(server.recv() == 1); + std::string expect{"test,t1=v1,t2= a1=="}; + push_double_arr_to_buffer(expect, arr_data, 1, test_shape) + .append(" 10000000\n"); + CHECK(server.msgs(0) == expect); +} + +#if __cplusplus >= 202002L +TEST_CASE("line_sender array span API") +{ + questdb::ingress::test::mock_server server; + questdb::ingress::opts opts{ + questdb::ingress::protocol::tcp, + std::string("127.0.0.1"), + std::to_string(server.port())}; + opts.protocol_version(questdb::ingress::protocol_version::v2); + questdb::ingress::line_sender sender{opts}; + CHECK_FALSE(sender.must_close()); + server.accept(); + CHECK(server.recv() == 0); + + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + + std::vector arr_data = { + 48123.5, + 2.4, + 48124.0, + 1.8, + 48124.5, + 0.9, + 48122.5, + 3.1, + 48122.0, + 2.7, + 48121.5, + 4.3}; + std::span data_span = arr_data; + buffer.table("test") + .symbol("t1", "v1") + .symbol("t2", "") + .column("a1", data_span.subspan(1, 8)) + .at(questdb::ingress::timestamp_nanos{10000000}); + std::vector expect_arr_data = { + 2.4, 48124.0, 1.8, 48124.5, 0.9, 48122.5, 3.1, 48122.0}; + + uintptr_t test_shape[] = {8}; + CHECK(server.recv() == 0); + CHECK(buffer.size() == 100); + sender.flush(buffer); + CHECK(server.recv() == 1); + std::string expect{"test,t1=v1,t2= a1=="}; + push_double_arr_to_buffer(expect, expect_arr_data, 1, test_shape) + .append(" 10000000\n"); + CHECK(server.msgs(0) == expect); +} +#endif + TEST_CASE("test multiple lines") { questdb::ingress::test::mock_server server; diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 22a3e6e0..158aefe5 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -836,6 +836,68 @@ class line_sender_buffer return *this; } + /** + * Records a 1-dimensional vector of double-precision values as array. + * + * QuestDB server version 8.4.0 or later is required for array support. + * + * @tparam T Element type (current only `double` is supported). + * + * @param name Column name. + * @param data Vector. + */ + template + line_sender_buffer& column( + column_name_view name, const std::vector& data) + { + static_assert( + std::is_same_v, + "Only double types are supported for arrays"); + may_init(); + uintptr_t array_shape[] = {data.size()}; + line_sender_error::wrapped_call( + ::line_sender_buffer_column_f64_arr_c_major, + _impl, + name._impl, + 1, + array_shape, + reinterpret_cast(data.data()), + data.size() * sizeof(T)); + return *this; + } + +#if __cplusplus >= 202002L + /** + * Records a 1-dimensional span of double-precision values as array. + * + * QuestDB server version 8.4.0 or later is required for array support. + * + * @tparam T Element type (current only `double` is supported). + * + * @param name Column name. + * @param data Vector. + */ + template + line_sender_buffer& column( + column_name_view name, const std::span data) + { + static_assert( + std::is_same_v, + "Only double types are supported for arrays"); + may_init(); + uintptr_t array_shape[] = {data.size()}; + line_sender_error::wrapped_call( + ::line_sender_buffer_column_f64_arr_c_major, + _impl, + name._impl, + 1, + array_shape, + reinterpret_cast(data.data()), + data.size() * sizeof(T)); + return *this; + } +#endif + /** * Record a string value for the given column. * @param name Column name. From 28bd22c610e116eb323ac5a94154e9cbb4ee7e06 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 11 Jun 2025 23:20:54 +0800 Subject: [PATCH 23/46] add inline(always) for strideArrayView Iterator --- questdb-rs-ffi/src/ndarr.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/questdb-rs-ffi/src/ndarr.rs b/questdb-rs-ffi/src/ndarr.rs index 71c9e689..00aff84e 100644 --- a/questdb-rs-ffi/src/ndarr.rs +++ b/questdb-rs-ffi/src/ndarr.rs @@ -167,6 +167,7 @@ where { type Item = &'a T; + #[inline(always)] fn next(&mut self) -> Option { if self.current_linear >= self.total_elements { return None; From 4a656bce01d1a32f25e9fe95de244fc7fd9e906d Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Tue, 17 Jun 2025 18:35:40 +0100 Subject: [PATCH 24/46] C++ API improvements --- cpp_test/test_line_sender.cpp | 47 ++-- ..._sender_cpp_example_array_byte_strides.cpp | 8 +- .../line_sender_cpp_example_array_c_major.cpp | 5 +- ..._sender_cpp_example_array_elem_strides.cpp | 7 +- include/questdb/ingress/line_sender.h | 12 +- include/questdb/ingress/line_sender.hpp | 259 +++++++++--------- 6 files changed, 185 insertions(+), 153 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 7898c739..18295bf6 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -218,15 +218,14 @@ TEST_CASE("line_sender c api basics") sizeof(arr_data), &err)); line_sender_column_name arr_name3 = QDB_COLUMN_NAME_LITERAL("a3"); - CHECK( - ::line_sender_buffer_column_f64_arr_c_major( - buffer, - arr_name3, - rank, - shape, - reinterpret_cast(arr_data.data()), - sizeof(arr_data), - &err)); + CHECK(::line_sender_buffer_column_f64_arr_c_major( + buffer, + arr_name3, + rank, + shape, + reinterpret_cast(arr_data.data()), + sizeof(arr_data), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); CHECK(::line_sender_buffer_size(buffer) == 382); @@ -307,20 +306,30 @@ TEST_CASE("line_sender c++ api basics") 48121.5, 4.3}; intptr_t elem_strides[] = {6, 2, 1}; + questdb::ingress::nd_array_strided_view< + double, + questdb::ingress::array_strides_size_mode::bytes> + a1{rank, shape, strides, arr_data.data(), arr_data.size()}; + questdb::ingress::nd_array_strided_view< + double, + questdb::ingress::array_strides_size_mode::elems> + a2{rank, shape, elem_strides, arr_data.data(), arr_data.size()}; + questdb::ingress::nd_array_row_major_view a3{ + rank, shape, arr_data.data(), arr_data.size()}; + questdb::ingress::nd_array_strided_view< + double, + questdb::ingress::array_strides_size_mode::bytes> + a4{rank, shape, strides, arr_data.data(), arr_data.size()}; buffer.table("test") .symbol("t1", "v1") .symbol("t2", "") .column("f1", 0.5) - .column( - "a1", rank, shape, strides, arr_data) - .column( - "a2", rank, shape, elem_strides, arr_data) - .column("a3", rank, shape, arr_data) - .column( - "a4", rank, shape, strides, arr_data.data(), arr_data.size()) - .column( - "a5", rank, shape, elem_strides, arr_data.data(), arr_data.size()) - .column("a6", rank, shape, arr_data.data(), arr_data.size()) + .column("a1", a1) + .column("a2", a2) + .column("a3", a3) + .column("a4", a4) + .column("a5", a2) + .column("a6", a3) .at(questdb::ingress::timestamp_nanos{10000000}); CHECK(server.recv() == 0); diff --git a/examples/line_sender_cpp_example_array_byte_strides.cpp b/examples/line_sender_cpp_example_array_byte_strides.cpp index 0fc6f3ef..67139e51 100644 --- a/examples/line_sender_cpp_example_array_byte_strides.cpp +++ b/examples/line_sender_cpp_example_array_byte_strides.cpp @@ -36,11 +36,15 @@ static bool array_example(std::string_view host, std::string_view port) 48121.5, 4.3}; + questdb::ingress::nd_array_strided_view< + double, + questdb::ingress::array_strides_size_mode::bytes> + book_data{rank, shape.data(), strides.data(), arr_data.data(), arr_data.size()}; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_col, "BTC-USD"_utf8) - .column( - book_col, 3, shape.data(), strides.data(), arr_data) + .column(book_col, book_data) .at(questdb::ingress::timestamp_nanos::now()); sender.flush(buffer); return true; diff --git a/examples/line_sender_cpp_example_array_c_major.cpp b/examples/line_sender_cpp_example_array_c_major.cpp index f74c379c..538ec671 100644 --- a/examples/line_sender_cpp_example_array_c_major.cpp +++ b/examples/line_sender_cpp_example_array_c_major.cpp @@ -35,10 +35,13 @@ static bool array_example(std::string_view host, std::string_view port) 48121.5, 4.3}; + questdb::ingress::nd_array_row_major_view + book_data{rank, shape.data(), arr_data.data(), arr_data.size()}; + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_col, "BTC-USD"_utf8) - .column(book_col, 3, shape.data(), arr_data) + .column(book_col, book_data) .at(questdb::ingress::timestamp_nanos::now()); sender.flush(buffer); return true; diff --git a/examples/line_sender_cpp_example_array_elem_strides.cpp b/examples/line_sender_cpp_example_array_elem_strides.cpp index ea64e1b4..46cce74f 100644 --- a/examples/line_sender_cpp_example_array_elem_strides.cpp +++ b/examples/line_sender_cpp_example_array_elem_strides.cpp @@ -35,12 +35,15 @@ static bool array_example(std::string_view host, std::string_view port) 2.7, 48121.5, 4.3}; + questdb::ingress::nd_array_strided_view< + double, + questdb::ingress::array_strides_size_mode::elems> + book_data{3, shape.data(), strides.data(), arr_data.data(), arr_data.size()}; questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) .symbol(symbol_col, "BTC-USD"_utf8) - .column( - book_col, 3, shape.data(), strides.data(), arr_data) + .column(book_col, book_data) .at(questdb::ingress::timestamp_nanos::now()); sender.flush(buffer); return true; diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 81791067..360f4410 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -534,8 +534,10 @@ bool line_sender_buffer_column_f64_arr_c_major( * @param[in] rank Number of dimensions of the array. * @param[in] shape Array of dimension sizes (length = `rank`). * Each element must be a positive integer. - * @param[in] strides Array strides, in the unit of bytes. Strides can be negative. - * @param[in] data_buffer Array data, laid out according to the provided shape and strides. + * @param[in] strides Array strides, in the unit of bytes. Strides can be + * negative. + * @param[in] data_buffer Array data, laid out according to the provided shape + * and strides. * @param[in] data_buffer_len Length of the array data block in bytes. * @param[out] err_out Set to an error object on failure (if non-NULL). * @return true on success, false on error. @@ -564,8 +566,10 @@ bool line_sender_buffer_column_f64_arr_byte_strides( * @param[in] rank Number of dimensions of the array. * @param[in] shape Array of dimension sizes (length = `rank`). * Each element must be a positive integer. - * @param[in] strides Array strides, in the unit of elements. Strides can be negative. - * @param[in] data_buffer Array data, laid out according to the provided shape and strides. + * @param[in] strides Array strides, in the unit of elements. Strides can be + * negative. + * @param[in] data_buffer Array data, laid out according to the provided shape + * and strides. * @param[in] data_buffer_len Length of the array data block in bytes. * @param[out] err_out Set to an error object on failure (if non-NULL). * @return true on success, false on error. diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 6cc4d4e4..dcc6ff8f 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -408,6 +408,100 @@ class buffer_view final }; #endif +template +class nd_array_strided_view +{ +public: + using element_type = T; + static constexpr array_strides_size_mode stride_size_mode = M; + + nd_array_strided_view( + size_t rank, + const uintptr_t* shape, + const intptr_t* strides, + const T* data, + size_t data_size) + : _rank{rank} + , _shape{shape} + , _strides{strides} + , _data{data} + , _data_size{data_size} + { + } + + size_t rank() const + { + return _rank; + } + + const uintptr_t* shape() const + { + return _shape; + } + + const intptr_t* strides() const + { + return _strides; + } + + const T* data() const + { + return _data; + } + + size_t data_size() const + { + return _data_size; + } + +private: + size_t _rank; + const uintptr_t* _shape; + const intptr_t* _strides; + const T* _data; + size_t _data_size; +}; + +template +class nd_array_row_major_view +{ +public: + using element_type = T; + + nd_array_row_major_view( + size_t rank, const uintptr_t* shape, const T* data, size_t data_size) + : _rank{rank} + , _shape{shape} + , _data{data} + , _data_size{data_size} + { + } + + size_t rank() const + { + return _rank; + } + const uintptr_t* shape() const + { + return _shape; + } + const T* data() const + { + return _data; + } + + size_t data_size() const + { + return _data_size; + } + +private: + size_t _rank; + const uintptr_t* _shape; + const T* _data; + size_t _data_size; +}; + class line_sender_buffer { public: @@ -658,50 +752,44 @@ class line_sender_buffer * * QuestDB server version 8.4.0 or later is required for array support. * - * @tparam L Array stride size mode (bytes or elements). * @tparam T Element type (current only `double` is supported). - * @tparam N Number of elements in the flat data array + * @tparam M Array stride size mode (bytes or elements). * * @param name Column name. - * @param shape Array dimensions (e.g., [2,3] for a 2x3 matrix). - * @param strides Strides for each dimension, in the unit specified by `B`. - * @param data Array data. + * @param data Multi-dimensional array. */ - template + template line_sender_buffer& column( - column_name_view name, - const size_t rank, - const uintptr_t* shape, - const intptr_t* strides, - const std::array& data) + column_name_view name, const nd_array_strided_view& array) { static_assert( std::is_same_v, "Only double types are supported for arrays"); may_init(); - switch (L) + switch (M) { case array_strides_size_mode::bytes: line_sender_error::wrapped_call( ::line_sender_buffer_column_f64_arr_byte_strides, _impl, name._impl, - rank, - shape, - strides, - reinterpret_cast(data.data()), - sizeof(double) * N); + array.rank(), + array.shape(), + array.strides(), + reinterpret_cast(array.data()), + sizeof(T) * array.data_size()); break; case array_strides_size_mode::elems: line_sender_error::wrapped_call( ::line_sender_buffer_column_f64_arr_elem_strides, _impl, name._impl, - rank, - shape, - strides, - reinterpret_cast(data.data()), - sizeof(double) * N); + array.rank(), + array.shape(), + array.strides(), + reinterpret_cast(array.data()), + sizeof(T) * array.data_size()); + break; } return *this; } @@ -720,107 +808,9 @@ class line_sender_buffer * @param shape Array dimensions (e.g., [2,3] for a 2x3 matrix). * @param data Array data. */ - template - line_sender_buffer& column( - column_name_view name, - const size_t rank, - const size_t* shape, - const std::array& data) - { - static_assert( - std::is_same_v, - "Only double types are supported for arrays"); - may_init(); - line_sender_error::wrapped_call( - ::line_sender_buffer_column_f64_arr_c_major, - _impl, - name._impl, - rank, - shape, - reinterpret_cast(data.data()), - sizeof(double) * N); - return *this; - } - - /** - * Records a multidimensional array of double-precision values with - * configurable stride mode. - * - * QuestDB server version 8.4.0 or later is required for array support. - * - * @tparam L Array stride size mode (bytes or elements). - * @tparam T Element type (current only `double` is supported). - * - * @param name Column name. - * @param rank Number of dimensions of the array. - * @param shape Array dimensions. Example: [2,3] for 2x3 matrix. - * @param strides Array strides. Step between consecutive elements in each - * dimension. - * @param data Raw pointer to the start of the array data. - * @param elem_count Total element of the array. - */ - template - line_sender_buffer& column( - column_name_view name, - const size_t rank, - const uintptr_t* shape, - const intptr_t* strides, - const T* data, - size_t elem_count) - { - static_assert( - std::is_same_v, - "Only double types are supported for arrays"); - may_init(); - switch (L) - { - case array_strides_size_mode::bytes: - line_sender_error::wrapped_call( - ::line_sender_buffer_column_f64_arr_byte_strides, - _impl, - name._impl, - rank, - shape, - strides, - reinterpret_cast(data), - elem_count * sizeof(T)); - break; - case array_strides_size_mode::elems: - line_sender_error::wrapped_call( - ::line_sender_buffer_column_f64_arr_elem_strides, - _impl, - name._impl, - rank, - shape, - strides, - reinterpret_cast(data), - elem_count * sizeof(T)); - } - return *this; - } - - /** - * Records a multidimensional array of double-precision values in C-major - * (row-major) layout layout. - * - * QuestDB server version 8.4.0 or later is required for array support. - * - * @tparam T Element type (current only `double` is supported). - * - * @param name Column name. - * @param rank Number of dimensions in the array. - * @param shape Array dimensions. Example: [2,3] for 2x3 matrix. - * @param data Raw pointer to the first element of the flat data - * array. - * @param elem_count Total element of the array. - */ template line_sender_buffer& column( - column_name_view name, - const size_t rank, - const size_t* shape, - const T* data, - size_t elem_count) + column_name_view name, const nd_array_row_major_view& array) { static_assert( std::is_same_v, @@ -830,10 +820,10 @@ class line_sender_buffer ::line_sender_buffer_column_f64_arr_c_major, _impl, name._impl, - rank, - shape, - reinterpret_cast(data), - elem_count * sizeof(T)); + array.rank(), + array.shape(), + reinterpret_cast(array.data()), + sizeof(double) * array.data_size()); return *this; } @@ -1669,4 +1659,23 @@ class line_sender ::line_sender* _impl; }; +// template +// line_sender_buffer& column(column_name_view name, const ArrayT& array) +// { +// const auto array_view = questdb::ingress::array::to_view(array); +// return column(name, array_view); +// } + +// namespace array +// { +// template +// questdb::ingress::nd_array_row_major_view to_view( +// const std::array& array) noexcept +// { +// return questdb::ingress::nd_array_row_major_view{array.data(), 1, +// {N}}; +// } + +// } + } // namespace questdb::ingress From aa918b44cd5d7da59fe1c3ca55eb823fb7b8d8ff Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 18 Jun 2025 12:35:00 +0800 Subject: [PATCH 25/46] little refactor c api for array. --- cpp_test/test_line_sender.cpp | 65 +++++---- ...line_sender_c_example_array_byte_strides.c | 4 +- .../line_sender_c_example_array_c_major.c | 4 +- include/questdb/ingress/line_sender.h | 24 +-- include/questdb/ingress/line_sender.hpp | 50 +++++-- questdb-rs-ffi/src/lib.rs | 57 ++++---- questdb-rs-ffi/src/ndarr.rs | 137 ++++++++---------- system_test/questdb_line_sender.py | 13 +- 8 files changed, 185 insertions(+), 169 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 18295bf6..f798c582 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -196,36 +196,39 @@ TEST_CASE("line_sender c api basics") 2.7, 48121.5, 4.3}; - CHECK(::line_sender_buffer_column_f64_arr_byte_strides( - buffer, - arr_name, - rank, - shape, - strides, - reinterpret_cast(arr_data.data()), - sizeof(arr_data), - &err)); + CHECK( + ::line_sender_buffer_column_f64_arr_byte_strides( + buffer, + arr_name, + rank, + shape, + strides, + arr_data.data(), + arr_data.size(), + &err)); line_sender_column_name arr_name2 = QDB_COLUMN_NAME_LITERAL("a2"); intptr_t elem_strides[] = {6, 2, 1}; - CHECK(::line_sender_buffer_column_f64_arr_elem_strides( - buffer, - arr_name2, - rank, - shape, - elem_strides, - reinterpret_cast(arr_data.data()), - sizeof(arr_data), - &err)); + CHECK( + ::line_sender_buffer_column_f64_arr_elem_strides( + buffer, + arr_name2, + rank, + shape, + elem_strides, + arr_data.data(), + arr_data.size(), + &err)); line_sender_column_name arr_name3 = QDB_COLUMN_NAME_LITERAL("a3"); - CHECK(::line_sender_buffer_column_f64_arr_c_major( - buffer, - arr_name3, - rank, - shape, - reinterpret_cast(arr_data.data()), - sizeof(arr_data), - &err)); + CHECK( + ::line_sender_buffer_column_f64_arr_c_major( + buffer, + arr_name3, + rank, + shape, + arr_data.data(), + arr_data.size(), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); CHECK(::line_sender_buffer_size(buffer) == 382); @@ -328,22 +331,22 @@ TEST_CASE("line_sender c++ api basics") .column("a2", a2) .column("a3", a3) .column("a4", a4) - .column("a5", a2) - .column("a6", a3) + .column("a5", arr_data) .at(questdb::ingress::timestamp_nanos{10000000}); CHECK(server.recv() == 0); - CHECK(buffer.size() == 734); + CHECK(buffer.size() == 610); sender.flush(buffer); CHECK(server.recv() == 1); std::string expect{"test,t1=v1,t2= f1=="}; + uintptr_t shapes_1dim[] = {12}; push_double_to_buffer(expect, 0.5).append(",a1=="); push_double_arr_to_buffer(expect, arr_data, 3, shape).append(",a2=="); push_double_arr_to_buffer(expect, arr_data, 3, shape).append(",a3=="); push_double_arr_to_buffer(expect, arr_data, 3, shape).append(",a4=="); push_double_arr_to_buffer(expect, arr_data, 3, shape).append(",a5=="); - push_double_arr_to_buffer(expect, arr_data, 3, shape).append(",a6=="); - push_double_arr_to_buffer(expect, arr_data, 3, shape).append(" 10000000\n"); + push_double_arr_to_buffer(expect, arr_data, 1, shapes_1dim) + .append(" 10000000\n"); CHECK(server.msgs(0) == expect); } diff --git a/examples/line_sender_c_example_array_byte_strides.c b/examples/line_sender_c_example_array_byte_strides.c index aa89987d..8fc2da30 100644 --- a/examples/line_sender_c_example_array_byte_strides.c +++ b/examples/line_sender_c_example_array_byte_strides.c @@ -69,8 +69,8 @@ static bool example(const char* host, const char* port) array_rank, array_shape, array_strides, - (const uint8_t*)array_data, - sizeof(array_data), + array_data, + sizeof(array_data) / sizeof(array_data[0]), &err)) goto on_error; diff --git a/examples/line_sender_c_example_array_c_major.c b/examples/line_sender_c_example_array_c_major.c index 2e8b31e5..e9ce052e 100644 --- a/examples/line_sender_c_example_array_c_major.c +++ b/examples/line_sender_c_example_array_c_major.c @@ -66,8 +66,8 @@ static bool example(const char* host, const char* port) book_col, array_rank, array_shape, - (const uint8_t*)array_data, - sizeof(array_data), + array_data, + sizeof(array_data) / sizeof(array_data[0]), &err)) goto on_error; diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index 360f4410..c134d22e 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -506,8 +506,8 @@ bool line_sender_buffer_column_str( * @param[in] rank Number of dimensions of the array. * @param[in] shape Array of dimension sizes (length = `rank`). * Each element must be a positive integer. - * @param[in] data_buffer First array element data. - * @param[in] data_buffer_len Bytes length of the array data. + * @param[in] data First array element data. + * @param[in] data_len Element length of the array. * @param[out] err_out Set to an error object on failure (if non-NULL). * @return true on success, false on error. */ @@ -517,8 +517,8 @@ bool line_sender_buffer_column_f64_arr_c_major( line_sender_column_name name, size_t rank, const uintptr_t* shape, - const uint8_t* data_buffer, - size_t data_buffer_len, + const double* data, + size_t data_len, line_sender_error** err_out); /** @@ -536,9 +536,9 @@ bool line_sender_buffer_column_f64_arr_c_major( * Each element must be a positive integer. * @param[in] strides Array strides, in the unit of bytes. Strides can be * negative. - * @param[in] data_buffer Array data, laid out according to the provided shape + * @param[in] data Array data, laid out according to the provided shape * and strides. - * @param[in] data_buffer_len Length of the array data block in bytes. + * @param[in] data_len Element length of the array. * @param[out] err_out Set to an error object on failure (if non-NULL). * @return true on success, false on error. */ @@ -549,8 +549,8 @@ bool line_sender_buffer_column_f64_arr_byte_strides( size_t rank, const uintptr_t* shape, const intptr_t* strides, - const uint8_t* data_buffer, - size_t data_buffer_len, + const double* data, + size_t data_len, line_sender_error** err_out); /** @@ -568,9 +568,9 @@ bool line_sender_buffer_column_f64_arr_byte_strides( * Each element must be a positive integer. * @param[in] strides Array strides, in the unit of elements. Strides can be * negative. - * @param[in] data_buffer Array data, laid out according to the provided shape + * @param[in] data Array data, laid out according to the provided shape * and strides. - * @param[in] data_buffer_len Length of the array data block in bytes. + * @param[in] data_len Element length of the array. * @param[out] err_out Set to an error object on failure (if non-NULL). * @return true on success, false on error. */ @@ -581,8 +581,8 @@ bool line_sender_buffer_column_f64_arr_elem_strides( size_t rank, const uintptr_t* shape, const intptr_t* strides, - const uint8_t* data_buffer, - size_t data_buffer_len, + const double* data, + size_t data_len, line_sender_error** err_out); /** diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index dcc6ff8f..54b99a8c 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -776,8 +776,8 @@ class line_sender_buffer array.rank(), array.shape(), array.strides(), - reinterpret_cast(array.data()), - sizeof(T) * array.data_size()); + array.data(), + array.data_size()); break; case array_strides_size_mode::elems: line_sender_error::wrapped_call( @@ -787,8 +787,8 @@ class line_sender_buffer array.rank(), array.shape(), array.strides(), - reinterpret_cast(array.data()), - sizeof(T) * array.data_size()); + array.data(), + array.data_size()); break; } return *this; @@ -822,8 +822,8 @@ class line_sender_buffer name._impl, array.rank(), array.shape(), - reinterpret_cast(array.data()), - sizeof(double) * array.data_size()); + array.data(), + array.data_size()); return *this; } @@ -852,8 +852,38 @@ class line_sender_buffer name._impl, 1, array_shape, - reinterpret_cast(data.data()), - data.size() * sizeof(T)); + data.data(), + data.size()); + return *this; + } + + /** + * Records a 1-dimensional std::array of double-precision values as array. + * + * QuestDB server version 8.4.0 or later is required for array support. + * + * @tparam T Element type (current only `double` is supported). + * + * @param name Column name. + * @param data array. + */ + template + line_sender_buffer& column( + column_name_view name, const std::array& data) + { + static_assert( + std::is_same_v, + "Only double types are supported for arrays"); + may_init(); + uintptr_t shape[] = {N}; + line_sender_error::wrapped_call( + ::line_sender_buffer_column_f64_arr_c_major, + _impl, + name._impl, + 1, + shape, + data.data(), + N); return *this; } @@ -883,8 +913,8 @@ class line_sender_buffer name._impl, 1, array_shape, - reinterpret_cast(data.data()), - data.size() * sizeof(T)); + data.data(), + data.size()); return *this; } #endif diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index baab0cba..5b54946a 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -77,18 +77,13 @@ macro_rules! new_stride_array { $n:literal, $shape:expr, $strides:expr, - $data_buffer:expr, - $data_buffer_len:expr, + $data:expr, + $data_len:expr, $err_out:expr, $buffer:expr, $name:expr ) => {{ - let view = match StrideArrayView::::new( - $shape, - $strides, - $data_buffer, - $data_buffer_len, - ) { + let view = match StrideArrayView::::new($shape, $strides, $data, $data_len) { Ok(value) => value, Err(err) => { let err_ptr = Box::into_raw(Box::new(line_sender_error(err))); @@ -105,7 +100,7 @@ macro_rules! new_stride_array { } macro_rules! generate_array_dims_branches { - ($rank:expr, $m:literal, $shape:expr, $strides:expr, $data_buffer:expr, $data_buffer_len:expr, $err_out:expr, $buffer:expr, $name:expr => $($n:literal),*) => { + ($rank:expr, $m:literal, $shape:expr, $strides:expr, $data:expr, $data_len:expr, $err_out:expr, $buffer:expr, $name:expr => $($n:literal),*) => { match $rank { 0 => { let err = fmt_error!( @@ -123,8 +118,8 @@ macro_rules! generate_array_dims_branches { $n, $shape, $strides, - $data_buffer, - $data_buffer_len, + $data, + $data_len, $err_out, $buffer, $name @@ -945,13 +940,13 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( /// @param[in] name Column name. /// @param[in] rank Array dims. /// @param[in] shape Array shape. -/// @param[in] data_buffer Array **first element** data memory ptr. -/// @param[in] data_buffer_len Array data memory length. +/// @param[in] data Array **first element** data memory ptr. +/// @param[in] data_len Array data length. /// @param[out] err_out Set on error. /// # Safety /// - All pointer parameters must be valid and non-null /// - shape must point to an array of `rank` integers -/// - data_buffer must point to a buffer of size `data_buffer_len` bytes +/// - data must point to a buffer of size `data_len` f64 elements. /// - QuestDB server version 8.4.0 or later is required for array support. #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_c_major( @@ -959,13 +954,13 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_c_major( name: line_sender_column_name, rank: size_t, shape: *const usize, - data_buffer: *const u8, - data_buffer_len: size_t, + data: *const f64, + data_len: size_t, err_out: *mut *mut line_sender_error, ) -> bool { let buffer = unwrap_buffer_mut(buffer); let name = name.as_name(); - let view = match CMajorArrayView::::new(rank, shape, data_buffer, data_buffer_len) { + let view = match CMajorArrayView::::new(rank, shape, data, data_len) { Ok(value) => value, Err(err) => { let err_ptr = Box::into_raw(Box::new(line_sender_error(err))); @@ -989,13 +984,13 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_c_major( /// @param[in] rank Array dims. /// @param[in] shape Array shape. /// @param[in] strides Array strides, represent byte offsets between elements along each dimension. -/// @param[in] data_buffer Array **first element** data memory ptr. -/// @param[in] data_buffer_len Array data memory length. +/// @param[in] data Array **first element** data memory ptr. +/// @param[in] data_len Array data element length. /// @param[out] err_out Set on error. /// # Safety /// - All pointer parameters must be valid and non-null /// - shape must point to an array of `rank` integers -/// - data_buffer must point to a buffer of size `data_buffer_len` bytes +/// - data must point to a buffer of size `data_len` f64 elements. /// - QuestDB server version 8.4.0 or later is required for array support. #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_byte_strides( @@ -1004,8 +999,8 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_byte_strides( rank: size_t, shape: *const usize, strides: *const isize, - data_buffer: *const u8, - data_buffer_len: size_t, + data: *const f64, + data_len: size_t, err_out: *mut *mut line_sender_error, ) -> bool { let buffer = unwrap_buffer_mut(buffer); @@ -1015,8 +1010,8 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_byte_strides( 1, shape, strides, - data_buffer, - data_buffer_len, + data, + data_len, err_out, buffer, name @@ -1035,13 +1030,13 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_byte_strides( /// @param[in] rank Array dims. /// @param[in] shape Array shape. /// @param[in] strides Array strides, represent element counts between elements along each dimension. -/// @param[in] data_buffer Array **first element** data memory ptr. -/// @param[in] data_buffer_len Array data memory length. +/// @param[in] data Array **first element** data memory ptr. +/// @param[in] data_len Array data element length. /// @param[out] err_out Set on error. /// # Safety /// - All pointer parameters must be valid and non-null /// - shape must point to an array of `rank` integers -/// - data_buffer must point to a buffer of size `data_buffer_len` bytes +/// - data must point to a buffer of size `data_len` f64 elements. /// - QuestDB server version 8.4.0 or later is required for array support. #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_elem_strides( @@ -1050,8 +1045,8 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_elem_strides( rank: size_t, shape: *const usize, strides: *const isize, - data_buffer: *const u8, - data_buffer_len: size_t, + data: *const f64, + data_len: size_t, err_out: *mut *mut line_sender_error, ) -> bool { let buffer = unwrap_buffer_mut(buffer); @@ -1061,8 +1056,8 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_elem_strides( 8, shape, strides, - data_buffer, - data_buffer_len, + data, + data_len, err_out, buffer, name diff --git a/questdb-rs-ffi/src/ndarr.rs b/questdb-rs-ffi/src/ndarr.rs index 00aff84e..f8e5ec57 100644 --- a/questdb-rs-ffi/src/ndarr.rs +++ b/questdb-rs-ffi/src/ndarr.rs @@ -35,7 +35,7 @@ use std::slice; pub struct StrideArrayView<'a, T, const N: isize, const D: usize> { shape: &'a [usize], strides: &'a [isize], - data: Option<&'a [u8]>, + data: Option<&'a [T]>, _marker: std::marker::PhantomData, } @@ -67,9 +67,10 @@ where fn as_slice(&self) -> Option<&[T]> { unsafe { - self.is_c_major().then_some(self.data.map(|data| { - slice::from_raw_parts(data.as_ptr() as *const T, data.len() / size_of::()) - })?) + self.is_c_major().then_some( + self.data + .map(|data| slice::from_raw_parts(data.as_ptr(), data.len()))?, + ) } } @@ -77,7 +78,7 @@ where // consider minus strides let base_ptr = match self.data { None => std::ptr::null(), - Some(data) => data.as_ptr(), + Some(data) => data.as_ptr() as *const u8, }; RowMajorIter { base_ptr, @@ -101,21 +102,21 @@ where /// - `strides` points to a valid array of at least `dims` elements /// - `data` points to a valid memory block of at least `data_len` bytes /// - Memory layout must satisfy: - /// 1. `data_len ≥ (shape[0]-1)*abs(strides[0]) + ... + (shape[n-1]-1)*abs(strides[n-1]) + size_of::()` - /// 2. All calculated offsets stay within `[0, data_len - size_of::()]` + /// 1. `len == (shape[0]-1)*abs(strides[0]) + ... + (shape[n-1]-1)*abs(strides[n-1])` + /// 2. All calculated offsets stay within `[0, data_len * size_of::() - size_of::()]` /// - Lifetime `'a` must outlive the view's usage /// - Strides are measured in bytes (not elements) pub unsafe fn new( shape: *const usize, strides: *const isize, - data: *const u8, - data_len: usize, + data: *const T, + len: usize, ) -> Result { - let shape = check_array_shape::(D, shape, data_len)?; + let shape = check_array_shape::(D, shape, len)?; let strides = slice::from_raw_parts(strides, D); let mut slice = None; - if data_len != 0 { - slice = Some(slice::from_raw_parts(data, data_len)); + if len != 0 { + slice = Some(slice::from_raw_parts(data, len)); } Ok(Self { shape, @@ -235,7 +236,7 @@ where pub struct CMajorArrayView<'a, T> { dims: usize, shape: &'a [usize], - data: Option<&'a [u8]>, + data: Option<&'a [T]>, _marker: std::marker::PhantomData, } @@ -266,16 +267,15 @@ where } fn as_slice(&self) -> Option<&[T]> { - self.data.map(|d| unsafe { - slice::from_raw_parts(d.as_ptr() as *const T, d.len() / size_of::()) - }) + self.data + .map(|d| unsafe { slice::from_raw_parts(d.as_ptr(), d.len()) }) } fn iter(&self) -> Self::Iter<'_> { let elem_size = self.shape.iter().product(); let count = 0; let data_ptr = match self.data { - Some(data) => data.as_ptr() as *const T, + Some(data) => data.as_ptr(), None => std::ptr::null_mut(), }; CMajorArrayViewIterator { @@ -322,11 +322,11 @@ where /// # Safety /// Caller must ensure all the following conditions: /// - `shape` points to a valid array of at least `dims` elements - /// - `data` points to a valid memory block of at least `data_len` bytes + /// - `data` points to a valid memory block of at least `data_len` elements pub unsafe fn new( dims: usize, shape: *const usize, - data: *const u8, + data: *const T, data_len: usize, ) -> Result { let shape = check_array_shape::(dims, shape, data_len)?; @@ -369,12 +369,13 @@ fn check_array_shape( .try_fold(std::mem::size_of::(), |acc, &dim| { acc.checked_mul(dim) .ok_or_else(|| fmt_error!(ArrayError, "Array buffer size too big")) - })?; + })? + / std::mem::size_of::(); if size != data_len { return Err(fmt_error!( ArrayError, - "Array buffer length mismatch (actual: {}, expected: {})", + "Array element length mismatch (actual: {}, expected: {})", data_len, size )); @@ -473,8 +474,8 @@ mod tests { StrideArrayView::new( [2, 2].as_ptr(), [2 * elem_size, elem_size].as_ptr(), - test_data.as_ptr() as *const u8, - test_data.len() * elem_size as usize, + test_data.as_ptr(), + test_data.len(), ) }?; let mut buffer = Buffer::new(ProtocolVersion::V2); @@ -510,15 +511,13 @@ mod tests { #[test] fn test_buffer_basic_write_with_elem_strides() -> TestResult { - let elem_size = std::mem::size_of::() as isize; - let test_data = [1.1, 2.2, 3.3, 4.4]; let array_view: StrideArrayView<'_, f64, 8, 2> = unsafe { StrideArrayView::new( [2, 2].as_ptr(), [2, 1].as_ptr(), - test_data.as_ptr() as *const u8, - test_data.len() * elem_size as usize, + test_data.as_ptr(), + test_data.len(), ) }?; let mut buffer = Buffer::new(ProtocolVersion::V2); @@ -576,23 +575,23 @@ mod tests { StrideArrayView::new( [1, 2].as_ptr(), [elem_size, elem_size].as_ptr(), - under_data.as_ptr() as *const u8, - under_data.len() * elem_size as usize, + under_data.as_ptr(), + under_data.len(), ) }; let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::ArrayError); assert!(err .msg() - .contains("Array buffer length mismatch (actual: 8, expected: 16)")); + .contains("Array element length mismatch (actual: 1, expected: 2)")); let over_data = [1.1, 2.2, 3.3]; let result: Result, Error> = unsafe { StrideArrayView::new( [1, 2].as_ptr(), [elem_size, elem_size].as_ptr(), - over_data.as_ptr() as *const u8, - over_data.len() * elem_size as usize, + over_data.as_ptr(), + over_data.len(), ) }; @@ -600,7 +599,7 @@ mod tests { assert_eq!(err.code(), ErrorCode::ArrayError); assert!(err .msg() - .contains("Array buffer length mismatch (actual: 24, expected: 16)")); + .contains("Array element length mismatch (actual: 3, expected: 2)")); Ok(()) } @@ -612,23 +611,23 @@ mod tests { StrideArrayView::new( [1, 2].as_ptr(), [1, 1].as_ptr(), - under_data.as_ptr() as *const u8, - under_data.len() * elem_size as usize, + under_data.as_ptr(), + under_data.len(), ) }; let err = result.unwrap_err(); assert_eq!(err.code(), ErrorCode::ArrayError); assert!(err .msg() - .contains("Array buffer length mismatch (actual: 8, expected: 16)")); + .contains("Array element length mismatch (actual: 1, expected: 2)")); let over_data = [1.1, 2.2, 3.3]; let result: Result, Error> = unsafe { StrideArrayView::new( [1, 2].as_ptr(), [elem_size, elem_size].as_ptr(), - over_data.as_ptr() as *const u8, - over_data.len() * elem_size as usize, + over_data.as_ptr(), + over_data.len(), ) }; @@ -636,7 +635,7 @@ mod tests { assert_eq!(err.code(), ErrorCode::ArrayError); assert!(err .msg() - .contains("Array buffer length mismatch (actual: 24, expected: 16)")); + .contains("Array element length mismatch (actual: 3, expected: 2)")); Ok(()) } @@ -651,8 +650,8 @@ mod tests { StrideArrayView::new( shape.as_ptr(), strides.as_ptr(), - col_major_data.as_ptr() as *const u8, - col_major_data.len() * elem_size as usize, + col_major_data.as_ptr(), + col_major_data.len(), ) }?; @@ -687,8 +686,8 @@ mod tests { StrideArrayView::new( shape.as_ptr(), strides.as_ptr(), - col_major_data.as_ptr() as *const u8, - col_major_data.len() * elem_size as usize, + col_major_data.as_ptr(), + col_major_data.len(), ) }?; @@ -720,8 +719,8 @@ mod tests { StrideArrayView::::new( &[3usize, 3] as *const usize, &[-24isize, 8] as *const isize, - (data.as_ptr() as *const u8).add(48), - data.len() * elem_size, + data.as_ptr().add(6), + data.len(), ) }?; let collected: Vec<_> = view.iter().copied().collect(); @@ -748,8 +747,8 @@ mod tests { StrideArrayView::::new( &[3usize, 3] as *const usize, &[-3isize, 1] as *const isize, - (data.as_ptr() as *const u8).add(48), - data.len() * elem_size, + data.as_ptr().add(6), + data.len(), ) }?; let collected: Vec<_> = view.iter().copied().collect(); @@ -781,12 +780,7 @@ mod tests { // single element array let single_data = [42.0]; let single_view: StrideArrayView<'_, f64, 1, 1> = unsafe { - StrideArrayView::new( - [1].as_ptr(), - [elem_size].as_ptr(), - single_data.as_ptr() as *const u8, - elem_size as usize, - ) + StrideArrayView::new([1].as_ptr(), [elem_size].as_ptr(), single_data.as_ptr(), 1) }?; let mut buf = vec![0u8; 8]; write_array_data(&single_view, &mut buf, 8).unwrap(); @@ -807,8 +801,8 @@ mod tests { StrideArrayView::::new( shape.as_ptr(), strides.as_ptr(), - test_data.as_ptr() as *const u8, - test_data.len() * size_of::(), + test_data.as_ptr(), + test_data.len(), ) }?; @@ -834,8 +828,8 @@ mod tests { StrideArrayView::() as isize }, 2>::new( shape.as_ptr(), strides.as_ptr(), - test_data.as_ptr() as *const u8, - test_data.len() * size_of::(), + test_data.as_ptr(), + test_data.len(), ) }?; @@ -862,8 +856,8 @@ mod tests { StrideArrayView::() as isize }, 2>::new( shape.as_ptr(), strides.as_ptr(), - test_data.as_ptr().add(11) as *const u8, - 4 * size_of::(), + test_data.as_ptr().add(11), + 4, ) }?; @@ -879,12 +873,7 @@ mod tests { fn test_c_major_array_basic() -> TestResult { let test_data = [1.1, 2.2, 3.3, 4.4]; let array_view: CMajorArrayView<'_, f64> = unsafe { - CMajorArrayView::new( - 2, - [2, 2].as_ptr(), - test_data.as_ptr() as *const u8, - test_data.len() * 8usize, - ) + CMajorArrayView::new(2, [2, 2].as_ptr(), test_data.as_ptr(), test_data.len()) }?; let mut buffer = Buffer::new(ProtocolVersion::V2); buffer.table("my_test")?; @@ -965,8 +954,8 @@ mod tests { StrideArrayView::new( shape.as_ptr(), strides.as_ptr(), - col_major_data.as_ptr() as *const u8, - col_major_data.len() * elem_size as usize, + col_major_data.as_ptr(), + col_major_data.len(), ) }?; assert_eq!(array_view.ndim(), 3); @@ -1017,7 +1006,6 @@ mod tests { #[test] fn test_stride_non_contiguous_elem_stride_3d() -> TestResult { - let elem_size = size_of::() as isize; let col_major_data = [ 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, ]; @@ -1027,8 +1015,8 @@ mod tests { StrideArrayView::new( shape.as_ptr(), strides.as_ptr(), - col_major_data.as_ptr() as *const u8, - col_major_data.len() * elem_size as usize, + col_major_data.as_ptr(), + col_major_data.len(), ) }?; assert_eq!(array_view.ndim(), 3); @@ -1095,8 +1083,8 @@ mod tests { StrideArrayView::new( shape.as_ptr(), strides.as_ptr(), - col_major_data.as_ptr() as *const u8, - col_major_data.len() * elem_size as usize, + col_major_data.as_ptr(), + col_major_data.len(), ) }?; @@ -1155,7 +1143,6 @@ mod tests { #[test] fn test_stride_non_contiguous_elem_stride_4d() -> TestResult { - let elem_size = size_of::() as isize; let col_major_data = [ 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, ]; @@ -1171,8 +1158,8 @@ mod tests { StrideArrayView::new( shape.as_ptr(), strides.as_ptr(), - col_major_data.as_ptr() as *const u8, - col_major_data.len() * elem_size as usize, + col_major_data.as_ptr(), + col_major_data.len(), ) }?; diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index d6c997e3..55e0fcb0 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -136,6 +136,7 @@ class c_line_sender_error(ctypes.Structure): c_line_sender_error_p = ctypes.POINTER(c_line_sender_error) c_line_sender_error_p_p = ctypes.POINTER(c_line_sender_error_p) c_uint8_p = ctypes.POINTER(c_uint8) +c_double_p = ctypes.POINTER(c_double) class c_line_sender_utf8(ctypes.Structure): @@ -297,7 +298,7 @@ def set_sig(fn, restype, *argtypes): c_size_t, c_size_t_p, c_ssize_t_p, - c_uint8_p, + c_double_p, c_size_t, c_line_sender_error_p_p) set_sig( @@ -307,7 +308,7 @@ def set_sig(fn, restype, *argtypes): c_line_sender_column_name, c_size_t, c_size_t_p, - c_uint8_p, + c_double_p, c_size_t, c_line_sender_error_p_p) set_sig( @@ -746,7 +747,7 @@ def _convert_tuple(tpl: tuple[int, ...], c_type: type, name: str) -> ctypes.POIN c_size_t(rank), c_shape, c_strides, - ctypes.cast(data, c_uint8_p), + ctypes.cast(data, c_double_p), c_size_t(length) ) @@ -771,7 +772,7 @@ def _convert_tuple(tpl: tuple[int, ...], c_type: type, name: str) -> ctypes.POIN _column_name(name), c_size_t(rank), c_shape, - ctypes.cast(data, c_uint8_p), + ctypes.cast(data, c_double_p), c_size_t(length) ) @@ -911,9 +912,9 @@ def column_f64_arr( if array.dtype != numpy.float64: raise ValueError('expect float64 array') if array.flags.c_contiguous: - self._buffer.column_f64_arr_c_major(name, array.ndim, array.shape, array.ctypes.data, array.nbytes) + self._buffer.column_f64_arr_c_major(name, array.ndim, array.shape, array.ctypes.data, array.size) else: - self._buffer.column_f64_arr(name, array.ndim, array.shape, array.strides, array.ctypes.data, array.nbytes) + self._buffer.column_f64_arr(name, array.ndim, array.shape, array.strides, array.ctypes.data, array.size) return self def at_now(self): From d2b14407550e4e6b561c1edbaa6df9c0a0b7c0e3 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 18 Jun 2025 12:56:16 +0800 Subject: [PATCH 26/46] fix c tests --- examples/line_sender_c_example_array_elem_strides.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/line_sender_c_example_array_elem_strides.c b/examples/line_sender_c_example_array_elem_strides.c index bcfa1a8b..96d02f68 100644 --- a/examples/line_sender_c_example_array_elem_strides.c +++ b/examples/line_sender_c_example_array_elem_strides.c @@ -69,8 +69,8 @@ static bool example(const char* host, const char* port) array_rank, array_shape, array_strides, - (const uint8_t*)array_data, - sizeof(array_data), + array_data, + sizeof(array_data) / sizeof(array_data[0]), &err)) goto on_error; From 71373cd3a76a58b7c5e532470831e446bf1a8196 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Wed, 18 Jun 2025 16:39:43 +0100 Subject: [PATCH 27/46] C++ API customization point for arrays --- cpp_test/test_line_sender.cpp | 23 +- ..._sender_cpp_example_array_byte_strides.cpp | 4 +- .../line_sender_cpp_example_array_c_major.cpp | 2 +- ..._sender_cpp_example_array_elem_strides.cpp | 4 +- include/questdb/ingress/line_sender.hpp | 273 +++++++++--------- 5 files changed, 159 insertions(+), 147 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index f798c582..8c0c53c9 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -309,20 +309,17 @@ TEST_CASE("line_sender c++ api basics") 48121.5, 4.3}; intptr_t elem_strides[] = {6, 2, 1}; - questdb::ingress::nd_array_strided_view< - double, - questdb::ingress::array_strides_size_mode::bytes> - a1{rank, shape, strides, arr_data.data(), arr_data.size()}; - questdb::ingress::nd_array_strided_view< - double, - questdb::ingress::array_strides_size_mode::elems> - a2{rank, shape, elem_strides, arr_data.data(), arr_data.size()}; - questdb::ingress::nd_array_row_major_view a3{ + questdb::ingress::array:: + strided_view + a1{rank, shape, strides, arr_data.data(), arr_data.size()}; + questdb::ingress::array:: + strided_view + a2{rank, shape, elem_strides, arr_data.data(), arr_data.size()}; + questdb::ingress::array::row_major_view a3{ rank, shape, arr_data.data(), arr_data.size()}; - questdb::ingress::nd_array_strided_view< - double, - questdb::ingress::array_strides_size_mode::bytes> - a4{rank, shape, strides, arr_data.data(), arr_data.size()}; + questdb::ingress::array:: + strided_view + a4{rank, shape, strides, arr_data.data(), arr_data.size()}; buffer.table("test") .symbol("t1", "v1") .symbol("t2", "") diff --git a/examples/line_sender_cpp_example_array_byte_strides.cpp b/examples/line_sender_cpp_example_array_byte_strides.cpp index 67139e51..ce1d11c6 100644 --- a/examples/line_sender_cpp_example_array_byte_strides.cpp +++ b/examples/line_sender_cpp_example_array_byte_strides.cpp @@ -36,9 +36,9 @@ static bool array_example(std::string_view host, std::string_view port) 48121.5, 4.3}; - questdb::ingress::nd_array_strided_view< + questdb::ingress::array::strided_view< double, - questdb::ingress::array_strides_size_mode::bytes> + questdb::ingress::array::strides_mode::bytes> book_data{rank, shape.data(), strides.data(), arr_data.data(), arr_data.size()}; questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); diff --git a/examples/line_sender_cpp_example_array_c_major.cpp b/examples/line_sender_cpp_example_array_c_major.cpp index 538ec671..637fc3da 100644 --- a/examples/line_sender_cpp_example_array_c_major.cpp +++ b/examples/line_sender_cpp_example_array_c_major.cpp @@ -35,7 +35,7 @@ static bool array_example(std::string_view host, std::string_view port) 48121.5, 4.3}; - questdb::ingress::nd_array_row_major_view + questdb::ingress::array::row_major_view book_data{rank, shape.data(), arr_data.data(), arr_data.size()}; questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); diff --git a/examples/line_sender_cpp_example_array_elem_strides.cpp b/examples/line_sender_cpp_example_array_elem_strides.cpp index 46cce74f..688bcf2a 100644 --- a/examples/line_sender_cpp_example_array_elem_strides.cpp +++ b/examples/line_sender_cpp_example_array_elem_strides.cpp @@ -35,9 +35,9 @@ static bool array_example(std::string_view host, std::string_view port) 2.7, 48121.5, 4.3}; - questdb::ingress::nd_array_strided_view< + questdb::ingress::array::strided_view< double, - questdb::ingress::array_strides_size_mode::elems> + questdb::ingress::array::strides_mode::elements> book_data{3, shape.data(), strides.data(), arr_data.data(), arr_data.size()}; questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 54b99a8c..586cdebb 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -113,15 +113,6 @@ enum class protocol_version v2 = 2, }; -enum class array_strides_size_mode -{ - /** Strides are provided in bytes */ - bytes, - - /** Strides are provided in elements */ - elems, -}; - /* Possible sources of the root certificates used to validate the server's TLS * certificate. */ enum class ca @@ -408,14 +399,25 @@ class buffer_view final }; #endif -template -class nd_array_strided_view +namespace array +{ +enum class strides_mode +{ + /** Strides are provided in bytes */ + bytes, + + /** Strides are provided in elements */ + elements, +}; + +template +class strided_view { public: using element_type = T; - static constexpr array_strides_size_mode stride_size_mode = M; + static constexpr strides_mode stride_size_mode = M; - nd_array_strided_view( + strided_view( size_t rank, const uintptr_t* shape, const intptr_t* strides, @@ -454,6 +456,11 @@ class nd_array_strided_view return _data_size; } + const strided_view& view() const + { + return *this; + } + private: size_t _rank; const uintptr_t* _shape; @@ -463,12 +470,12 @@ class nd_array_strided_view }; template -class nd_array_row_major_view +class row_major_view { public: using element_type = T; - nd_array_row_major_view( + row_major_view( size_t rank, const uintptr_t* shape, const T* data, size_t data_size) : _rank{rank} , _shape{shape} @@ -495,6 +502,11 @@ class nd_array_row_major_view return _data_size; } + const row_major_view& view() const + { + return *this; + } + private: size_t _rank; const uintptr_t* _shape; @@ -502,6 +514,92 @@ class nd_array_row_major_view size_t _data_size; }; +template +struct row_major_1d_holder +{ + uintptr_t shape[1]; + const T* data; + size_t size; + + row_major_1d_holder(const T* d, size_t s) : data(d), size(s) + { + shape[0] = static_cast(s); + } + + array::row_major_view view() const + { + return {1, shape, data, size}; + } +}; + +// template +// inline row_major_view::type> +// to_array_view_state_impl(const std::vector& vec) +// { +// uintptr_t shape[1] = {static_cast(vec.size())}; +// return {1, shape, vec.data(), vec.size()}; +// } + +// #if __cplusplus >= 202002L +// template +// inline row_major_view::type> +// to_array_view_state_impl(const std::span& span) +// { +// uintptr_t shape[1] = {static_cast(span.size())}; +// return {1, shape, span.data(), span.size()}; +// } +// #endif + +// template +// inline row_major_view::type> +// to_array_view_state_impl(const std::array& arr) +// { +// uintptr_t shape[1] = {static_cast(N)}; +// return {1, shape, arr.data(), N}; +// } + +template +inline auto to_array_view_state_impl(const std::vector& vec) { + return row_major_1d_holder::type>(vec.data(), vec.size()); +} + +#if __cplusplus >= 202002L +template +inline auto to_array_view_state_impl(const std::span& span) { + return row_major_1d_holder::type>(span.data(), span.size()); +} +#endif + +template +inline auto to_array_view_state_impl(const std::array& arr) { + return row_major_1d_holder::type>(arr.data(), N); +} + +/// Customization point to enable serialization of additonal types as arrays. +/// +/// Forwards to a namespace or ADL (König) lookup function. +/// The customized `to_array_view_state_impl` for your custom type can be placed +/// in either: +/// * The namespace of the type in question. +/// * In the `questdb::ingress::array` namespace. +/// +/// The function can either return a view object directly (either +/// `row_major_view` or `strided_view`), or, if you need to place some fields on +/// the stack, an object with a `.view()` method which returns a `const&` to one +/// of the two view types. Returning an object may be useful if you need to +/// "materialize" shape or strides information into contiguous memory. +struct to_array_view_state_fn { + template + auto operator()(const T& array) const { + // Implement your own `to_array_view_state_impl` as needed. + return to_array_view_state_impl(array); + } +}; + +inline constexpr to_array_view_state_fn to_array_view_state{}; + +} // namespace array + class line_sender_buffer { public: @@ -704,10 +802,29 @@ class line_sender_buffer } // Require specific overloads of `column` to avoid - // involuntary usage of the `bool` overload. - template + // involuntary usage of the `bool` overload or similar. + template < + typename T, + typename std::enable_if_t< + // Integral types that are NOT bool or int64_t + (std::is_integral_v> && + !std::is_same_v, bool> && + !std::is_same_v, int64_t>) || + + // Floating-point types that are NOT double + (std::is_floating_point_v> && + !std::is_same_v, double>) || + + // Pointer types (which can implicitly convert to bool) + std::is_pointer_v> + + , + int> = 0> line_sender_buffer& column(column_name_view name, T value) = delete; + // template + // line_sender_buffer& column(column_name_view name, T value) = delete; + /** * Record a boolean value for the given column. * @param name Column name. @@ -758,9 +875,9 @@ class line_sender_buffer * @param name Column name. * @param data Multi-dimensional array. */ - template + template line_sender_buffer& column( - column_name_view name, const nd_array_strided_view& array) + column_name_view name, const array::strided_view& array) { static_assert( std::is_same_v, @@ -768,7 +885,7 @@ class line_sender_buffer may_init(); switch (M) { - case array_strides_size_mode::bytes: + case array::strides_mode::bytes: line_sender_error::wrapped_call( ::line_sender_buffer_column_f64_arr_byte_strides, _impl, @@ -779,7 +896,7 @@ class line_sender_buffer array.data(), array.data_size()); break; - case array_strides_size_mode::elems: + case array::strides_mode::elements: line_sender_error::wrapped_call( ::line_sender_buffer_column_f64_arr_elem_strides, _impl, @@ -810,7 +927,7 @@ class line_sender_buffer */ template line_sender_buffer& column( - column_name_view name, const nd_array_row_major_view& array) + column_name_view name, const array::row_major_view& array) { static_assert( std::is_same_v, @@ -827,97 +944,14 @@ class line_sender_buffer return *this; } - /** - * Records a 1-dimensional vector of double-precision values as array. - * - * QuestDB server version 8.4.0 or later is required for array support. - * - * @tparam T Element type (current only `double` is supported). - * - * @param name Column name. - * @param data Vector. - */ - template - line_sender_buffer& column( - column_name_view name, const std::vector& data) - { - static_assert( - std::is_same_v, - "Only double types are supported for arrays"); - may_init(); - uintptr_t array_shape[] = {data.size()}; - line_sender_error::wrapped_call( - ::line_sender_buffer_column_f64_arr_c_major, - _impl, - name._impl, - 1, - array_shape, - data.data(), - data.size()); - return *this; - } - - /** - * Records a 1-dimensional std::array of double-precision values as array. - * - * QuestDB server version 8.4.0 or later is required for array support. - * - * @tparam T Element type (current only `double` is supported). - * - * @param name Column name. - * @param data array. - */ - template - line_sender_buffer& column( - column_name_view name, const std::array& data) - { - static_assert( - std::is_same_v, - "Only double types are supported for arrays"); - may_init(); - uintptr_t shape[] = {N}; - line_sender_error::wrapped_call( - ::line_sender_buffer_column_f64_arr_c_major, - _impl, - name._impl, - 1, - shape, - data.data(), - N); - return *this; - } - -#if __cplusplus >= 202002L - /** - * Records a 1-dimensional span of double-precision values as array. - * - * QuestDB server version 8.4.0 or later is required for array support. - * - * @tparam T Element type (current only `double` is supported). - * - * @param name Column name. - * @param data Vector. - */ - template - line_sender_buffer& column( - column_name_view name, const std::span data) + template + line_sender_buffer& column(column_name_view name, ToArrayViewT array) { - static_assert( - std::is_same_v, - "Only double types are supported for arrays"); may_init(); - uintptr_t array_shape[] = {data.size()}; - line_sender_error::wrapped_call( - ::line_sender_buffer_column_f64_arr_c_major, - _impl, - name._impl, - 1, - array_shape, - data.data(), - data.size()); - return *this; + const auto array_view_state = + questdb::ingress::array::to_array_view_state(array); + return column(name, array_view_state.view()); } -#endif /** * Record a string value for the given column. @@ -1689,23 +1723,4 @@ class line_sender ::line_sender* _impl; }; -// template -// line_sender_buffer& column(column_name_view name, const ArrayT& array) -// { -// const auto array_view = questdb::ingress::array::to_view(array); -// return column(name, array_view); -// } - -// namespace array -// { -// template -// questdb::ingress::nd_array_row_major_view to_view( -// const std::array& array) noexcept -// { -// return questdb::ingress::nd_array_row_major_view{array.data(), 1, -// {N}}; -// } - -// } - } // namespace questdb::ingress From cf62b6bd9f5d6aa4c11427a0c37f22c0a8ada279 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Wed, 18 Jun 2025 17:51:00 +0100 Subject: [PATCH 28/46] cleanup --- include/questdb/ingress/line_sender.hpp | 54 ++++++++----------------- 1 file changed, 17 insertions(+), 37 deletions(-) diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 586cdebb..e601b357 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -521,7 +521,9 @@ struct row_major_1d_holder const T* data; size_t size; - row_major_1d_holder(const T* d, size_t s) : data(d), size(s) + row_major_1d_holder(const T* d, size_t s) + : data(d) + , size(s) { shape[0] = static_cast(s); } @@ -532,46 +534,25 @@ struct row_major_1d_holder } }; -// template -// inline row_major_view::type> -// to_array_view_state_impl(const std::vector& vec) -// { -// uintptr_t shape[1] = {static_cast(vec.size())}; -// return {1, shape, vec.data(), vec.size()}; -// } - -// #if __cplusplus >= 202002L -// template -// inline row_major_view::type> -// to_array_view_state_impl(const std::span& span) -// { -// uintptr_t shape[1] = {static_cast(span.size())}; -// return {1, shape, span.data(), span.size()}; -// } -// #endif - -// template -// inline row_major_view::type> -// to_array_view_state_impl(const std::array& arr) -// { -// uintptr_t shape[1] = {static_cast(N)}; -// return {1, shape, arr.data(), N}; -// } - template -inline auto to_array_view_state_impl(const std::vector& vec) { - return row_major_1d_holder::type>(vec.data(), vec.size()); +inline auto to_array_view_state_impl(const std::vector& vec) +{ + return row_major_1d_holder::type>( + vec.data(), vec.size()); } #if __cplusplus >= 202002L template -inline auto to_array_view_state_impl(const std::span& span) { - return row_major_1d_holder::type>(span.data(), span.size()); +inline auto to_array_view_state_impl(const std::span& span) +{ + return row_major_1d_holder::type>( + span.data(), span.size()); } #endif template -inline auto to_array_view_state_impl(const std::array& arr) { +inline auto to_array_view_state_impl(const std::array& arr) +{ return row_major_1d_holder::type>(arr.data(), N); } @@ -588,9 +569,11 @@ inline auto to_array_view_state_impl(const std::array& arr) { /// the stack, an object with a `.view()` method which returns a `const&` to one /// of the two view types. Returning an object may be useful if you need to /// "materialize" shape or strides information into contiguous memory. -struct to_array_view_state_fn { +struct to_array_view_state_fn +{ template - auto operator()(const T& array) const { + auto operator()(const T& array) const + { // Implement your own `to_array_view_state_impl` as needed. return to_array_view_state_impl(array); } @@ -822,9 +805,6 @@ class line_sender_buffer int> = 0> line_sender_buffer& column(column_name_view name, T value) = delete; - // template - // line_sender_buffer& column(column_name_view name, T value) = delete; - /** * Record a boolean value for the given column. * @param name Column name. From c78ea1f7dc3f14e67c0dbcc560dc790dde3850fd Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 19 Jun 2025 15:45:14 +0100 Subject: [PATCH 29/46] doc clean-up --- cpp_test/test_line_sender.cpp | 55 +++++------ include/questdb/ingress/line_sender.hpp | 120 ++++++++++++++++++------ 2 files changed, 118 insertions(+), 57 deletions(-) diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 8c0c53c9..fce57f3b 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -196,39 +196,36 @@ TEST_CASE("line_sender c api basics") 2.7, 48121.5, 4.3}; - CHECK( - ::line_sender_buffer_column_f64_arr_byte_strides( - buffer, - arr_name, - rank, - shape, - strides, - arr_data.data(), - arr_data.size(), - &err)); + CHECK(::line_sender_buffer_column_f64_arr_byte_strides( + buffer, + arr_name, + rank, + shape, + strides, + arr_data.data(), + arr_data.size(), + &err)); line_sender_column_name arr_name2 = QDB_COLUMN_NAME_LITERAL("a2"); intptr_t elem_strides[] = {6, 2, 1}; - CHECK( - ::line_sender_buffer_column_f64_arr_elem_strides( - buffer, - arr_name2, - rank, - shape, - elem_strides, - arr_data.data(), - arr_data.size(), - &err)); + CHECK(::line_sender_buffer_column_f64_arr_elem_strides( + buffer, + arr_name2, + rank, + shape, + elem_strides, + arr_data.data(), + arr_data.size(), + &err)); line_sender_column_name arr_name3 = QDB_COLUMN_NAME_LITERAL("a3"); - CHECK( - ::line_sender_buffer_column_f64_arr_c_major( - buffer, - arr_name3, - rank, - shape, - arr_data.data(), - arr_data.size(), - &err)); + CHECK(::line_sender_buffer_column_f64_arr_c_major( + buffer, + arr_name3, + rank, + shape, + arr_data.data(), + arr_data.size(), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); CHECK(::line_sender_buffer_size(buffer) == 382); diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index e601b357..cd25fc33 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -350,42 +350,57 @@ class timestamp_nanos class buffer_view final { public: - /// @brief Default constructor. Creates an empty buffer view. + /** + * Default constructor. Creates an empty buffer view. + */ buffer_view() noexcept = default; - /// @brief Constructs a buffer view from raw byte data. - /// @param data Pointer to the underlying byte array (may be nullptr if - /// length=0). - /// @param length Number of bytes in the array. + /** + * Construct a buffer view from raw byte data. + * @param data Pointer to the underlying byte array (may be nullptr if + * length=0). + * @param length Number of bytes in the array. + */ constexpr buffer_view(const std::byte* data, size_t length) noexcept : buf(data) , len(length) { } - /// @brief Gets a pointer to the underlying byte array. - /// @return Const pointer to the data (may be nullptr if empty()). + /** + * Obtain a pointer to the underlying byte array. + * + * @return Const pointer to the data (may be nullptr if empty()). + */ constexpr const std::byte* data() const noexcept { return buf; } - /// @brief Gets the number of bytes in the view. - /// @return Size in bytes. + /** + * Obtain the number of bytes in the view. + * + * @return Size of the view in bytes. + */ constexpr size_t size() const noexcept { return len; } - /// @brief Checks if the view contains no bytes. - /// @return true if size() == 0. + /** + * Check if the buffer view is empty. + * @return true if the view has no bytes (size() == 0). + */ constexpr bool empty() const noexcept { return len == 0; } - /// @brief Checks byte-wise equality between two buffer views. - /// @return true if both views have identical size and byte content. + /** + * Check byte-wise if two buffer views are equal. + * @return true if both views have the same size and + * the same byte content. + */ friend bool operator==( const buffer_view& lhs, const buffer_view& rhs) noexcept { @@ -410,6 +425,19 @@ enum class strides_mode elements, }; +/** + * A view over a multi-dimensional array with custom strides. + * + * The strides can be expressed as bytes offsets or as element counts. + * The `rank` is the number of dimensions in the array, and the `shape` + * describes the size of each dimension. + * + * If the data is stored in a row-major order, it may be more convenient and + * efficient to use the `row_major_view` instead of `strided_view`. + * + * The `data` pointer must point to a contiguous block of memory that contains + * the array data. + */ template class strided_view { @@ -469,6 +497,21 @@ class strided_view size_t _data_size; }; +/** + * A view over a multi-dimensional array in row-major order. + * + * The `rank` is the number of dimensions in the array, and the `shape` + * describes the size of each dimension. + * + * The `data` pointer must point to a contiguous block of memory that contains + * the array data. + * + * If the source array is not stored in a row-major order, you may express + * the strides explicitly using the `strided_view` class. + * + * This class provides a simpler and more efficient interface for row-major + * arrays. + */ template class row_major_view { @@ -556,19 +599,21 @@ inline auto to_array_view_state_impl(const std::array& arr) return row_major_1d_holder::type>(arr.data(), N); } -/// Customization point to enable serialization of additonal types as arrays. -/// -/// Forwards to a namespace or ADL (König) lookup function. -/// The customized `to_array_view_state_impl` for your custom type can be placed -/// in either: -/// * The namespace of the type in question. -/// * In the `questdb::ingress::array` namespace. +/** + * Customization point to enable serialization of additonal types as arrays. + * + * Forwards to a namespace or ADL (König) lookup function. + * The customized `to_array_view_state_impl` for your custom type can be placed + * in either: + * * The namespace of the type in question. + * * In the `questdb::ingress::array` namespace. /// -/// The function can either return a view object directly (either -/// `row_major_view` or `strided_view`), or, if you need to place some fields on -/// the stack, an object with a `.view()` method which returns a `const&` to one -/// of the two view types. Returning an object may be useful if you need to -/// "materialize" shape or strides information into contiguous memory. + * The function can either return a view object directly (either + * `row_major_view` or `strided_view`), or, if you need to place some fields on + * the stack, an object with a `.view()` method which returns a `const&` to one + * "materialize" shape or strides information into contiguous memory. + * of the two view types. Returning an object may be useful if you need to + */ struct to_array_view_state_fn { template @@ -901,9 +946,7 @@ class line_sender_buffer * @tparam N Number of elements in the flat data array * * @param name Column name. - * @param rank Number of dimensions of the array. - * @param shape Array dimensions (e.g., [2,3] for a 2x3 matrix). - * @param data Array data. + * @param array Multi-dimensional array. */ template line_sender_buffer& column( @@ -924,6 +967,27 @@ class line_sender_buffer return *this; } + /** + * Record a multidimensional array of double-precision values. + * + * QuestDB server version 8.4.0 or later is required for array support. + * + * Use this method to record arrays of common or custom types such as + * `std::vector`, `std::span`, `std::array`, or custom types that can be + * converted to an array view. + * + * This overload uses a customization point to support additional types: + * If you need to support your additional types you may implement a + * `to_array_view_state_impl` function in the object's namespace (via ADL) + * or in the `questdb::ingress::array` namespace. + * Ensure that any additional customization points are included before + * `line_sender.hpp`. + * + * @tparam ToArrayViewT Type convertible to a custom object instance which + * can be converted to an array view. + * @param name Column name. + * @param array Multi-dimensional array. + */ template line_sender_buffer& column(column_name_view name, ToArrayViewT array) { From d6f2db80d41bd6ff8c847d4b0afe7e666b353b8f Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 19 Jun 2025 16:26:44 +0100 Subject: [PATCH 30/46] Added C++ array customisation point example --- CMakeLists.txt | 3 + .../line_sender_cpp_example_array_custom.cpp | 120 ++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 examples/line_sender_cpp_example_array_custom.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fcea964..ca613404 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,6 +142,9 @@ if (QUESTDB_TESTS_AND_EXAMPLES) compile_example( line_sender_cpp_example_array_elem_strides examples/line_sender_cpp_example_array_elem_strides.cpp) + compile_example( + line_sender_cpp_example_array_custom + examples/line_sender_cpp_example_array_custom.cpp) compile_example( line_sender_cpp_example_auth examples/line_sender_cpp_example_auth.cpp) diff --git a/examples/line_sender_cpp_example_array_custom.cpp b/examples/line_sender_cpp_example_array_custom.cpp new file mode 100644 index 00000000..17cadf38 --- /dev/null +++ b/examples/line_sender_cpp_example_array_custom.cpp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include + +using namespace std::literals::string_view_literals; +using namespace questdb::ingress::literals; + +struct ViewHolder { + std::array shape; + std::array strides; + const double* data; + size_t size; + questdb::ingress::array::strided_view view() const { + return {2, shape.data(), strides.data(), data, size}; + } +}; + +namespace custom_array { +class Matrix { +public: + Matrix(size_t rows, size_t cols) + : _rows{rows}, + _cols{cols}, + _data(rows * cols, std::numeric_limits::quiet_NaN()), + _row_stride{cols}, + _col_stride{1} {} + + size_t rows() const { return _rows; } + size_t cols() const { return _cols; } + size_t row_stride() const { return _row_stride; } + size_t col_stride() const { return _col_stride; } + const double* data() const { return _data.data(); } + size_t size() const { return _data.size(); } + + double get(size_t row, size_t col) const { + return _data[index(row, col)]; + } + + void set(size_t row, size_t col, double value) { + _data[index(row, col)] = value; + } + + void transpose() { + std::swap(_rows, _cols); + std::swap(_row_stride, _col_stride); + } + +private: + size_t _rows; + size_t _cols; + std::vector _data; + size_t _row_stride; + size_t _col_stride; + + size_t index(size_t row, size_t col) const { + if (row >= _rows || col >= _cols) { + throw std::out_of_range("Matrix indices out of bounds"); + } + return row * _row_stride + col * _col_stride; + } +}; + +// Customization point for QuestDB array API (discovered via König lookup) +// If you need to support a 3rd party type, put this function in the namespace +// of the type in question or in the `questdb::ingress::array` namespace +inline auto to_array_view_state_impl(const Matrix& m) { + return ViewHolder{ + {static_cast(m.rows()), static_cast(m.cols())}, + {static_cast(m.row_stride()), static_cast(m.col_stride())}, + m.data(), + m.size() + }; +} +} // namespace custom_array + +static bool array_example(std::string_view host, std::string_view port) +{ + using custom_array::Matrix; + try { + auto sender = questdb::ingress::line_sender::from_conf( + "http::addr=" + std::string{host} + ":" + std::string{port} + ";"); + const auto table_name = "cpp_matrix_demo"_tn; + const auto arr_col = "arr"_cn; + + Matrix m(2, 3); + m.set(0, 0, 1.1); m.set(0, 1, 2.2); m.set(0, 2, 3.3); + m.set(1, 0, 4.4); m.set(1, 1, 5.5); m.set(1, 2, 6.6); + + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) + .column(arr_col, m) + .at(questdb::ingress::timestamp_nanos::now()); + sender.flush(buffer); + + // Transpose and send again + m.transpose(); + buffer.clear(); + buffer.table(table_name) + .column(arr_col, m) + .at(questdb::ingress::timestamp_nanos::now()); + sender.flush(buffer); + return true; + } catch (const questdb::ingress::line_sender_error& err) { + std::cerr << "[ERROR] " << err.what() << std::endl; + return false; + } +} + +int main(int argc, const char* argv[]) +{ + auto host = "localhost"sv; + if (argc >= 2) + host = std::string_view{argv[1]}; + auto port = "9009"sv; + if (argc >= 3) + port = std::string_view{argv[2]}; + return !array_example(host, port); +} From 9f144e4c300a5014cf806f5d6e273afe77c7a685 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 19 Jun 2025 16:31:06 +0100 Subject: [PATCH 31/46] reformatted example code with clang-format --- ci/format_cpp.py | 7 ++ examples/concat.c | 16 ++-- examples/concat.h | 3 +- examples/line_sender_c_example.c | 19 ++-- ...line_sender_c_example_array_byte_strides.c | 6 +- .../line_sender_c_example_array_c_major.c | 6 +- ...line_sender_c_example_array_elem_strides.c | 6 +- examples/line_sender_c_example_auth.c | 22 +++-- examples/line_sender_c_example_auth_tls.c | 25 +++-- examples/line_sender_c_example_from_conf.c | 12 +-- examples/line_sender_c_example_from_env.c | 8 +- examples/line_sender_c_example_http.c | 14 +-- examples/line_sender_c_example_tls_ca.c | 33 ++++--- ..._sender_cpp_example_array_byte_strides.cpp | 12 ++- .../line_sender_cpp_example_array_c_major.cpp | 4 +- .../line_sender_cpp_example_array_custom.cpp | 95 +++++++++++++------ ..._sender_cpp_example_array_elem_strides.cpp | 7 +- 17 files changed, 195 insertions(+), 100 deletions(-) diff --git a/ci/format_cpp.py b/ci/format_cpp.py index 79d1257e..f39dd3aa 100755 --- a/ci/format_cpp.py +++ b/ci/format_cpp.py @@ -8,6 +8,7 @@ import sys sys.dont_write_bytecode = True import subprocess +import glob FILES = [ 'include/questdb/ingress/line_sender.h', @@ -18,6 +19,12 @@ 'cpp_test/test_line_sender.cpp', ] +# Also include all examples. +FILES += glob.glob('examples/*.c') +FILES += glob.glob('examples/*.cpp') +FILES += glob.glob('examples/*.h') +FILES += glob.glob('examples/*.hpp') + if __name__ == '__main__': check_mode = '--check' in sys.argv command = [ diff --git a/examples/concat.c b/examples/concat.c index 5ea5de8f..e22729ce 100644 --- a/examples/concat.c +++ b/examples/concat.c @@ -1,19 +1,23 @@ #include "concat.h" -/** Concatenate a list of nul-terminated strings. Pass an extra NULL arg to terminate. */ -char* concat_(const char* first, ...) { +/** Concatenate a list of nul-terminated strings. Pass an extra NULL arg to + * terminate. */ +char* concat_(const char* first, ...) +{ va_list args; size_t total_len = strlen(first) + 1; va_start(args, first); const char* str; - while((str = va_arg(args, char*)) != NULL) total_len += strlen(str); + while ((str = va_arg(args, char*)) != NULL) + total_len += strlen(str); va_end(args); char* result = calloc(total_len, sizeof(char)); - if(!result) return NULL; + if (!result) + return NULL; strcpy(result, first); va_start(args, first); - while((str = va_arg(args, char*)) != NULL) strcat(result, str); + while ((str = va_arg(args, char*)) != NULL) + strcat(result, str); va_end(args); return result; } - diff --git a/examples/concat.h b/examples/concat.h index af18d88d..2ba32fbf 100644 --- a/examples/concat.h +++ b/examples/concat.h @@ -7,5 +7,6 @@ char* concat_(const char* first, ...); -// A macro that passes the list of arguments to concat_ and adds a NULL terminator. +// A macro that passes the list of arguments to concat_ and adds a NULL +// terminator. #define concat(...) concat_(__VA_ARGS__, NULL) diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index 01796664..1dfbe77f 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -9,13 +9,16 @@ static bool example(const char* host, const char* port) line_sender_error* err = NULL; line_sender* sender = NULL; line_sender_buffer* buffer = NULL; - char* conf_str = concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); - if (!conf_str) { + char* conf_str = + concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); + if (!conf_str) + { fprintf(stderr, "Could not concatenate configuration string.\n"); return false; } - line_sender_utf8 conf_str_utf8 = { 0, NULL }; - if (!line_sender_utf8_init(&conf_str_utf8, strlen(conf_str), conf_str, &err)) + line_sender_utf8 conf_str_utf8 = {0, NULL}; + if (!line_sender_utf8_init( + &conf_str_utf8, strlen(conf_str), conf_str, &err)) goto on_error; sender = line_sender_from_conf(conf_str_utf8, &err); @@ -26,7 +29,7 @@ static bool example(const char* host, const char* port) conf_str = NULL; buffer = line_sender_buffer_new_for_sender(sender); - line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. + line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid @@ -37,7 +40,6 @@ static bool example(const char* host, const char* port) line_sender_column_name price_name = QDB_COLUMN_NAME_LITERAL("price"); line_sender_column_name amount_name = QDB_COLUMN_NAME_LITERAL("amount"); - if (!line_sender_buffer_table(buffer, table_name, &err)) goto on_error; @@ -74,7 +76,7 @@ static bool example(const char* host, const char* port) return true; -on_error: ; +on_error:; size_t err_len = 0; const char* err_msg = line_sender_error_msg(err, &err_len); fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); @@ -94,7 +96,8 @@ static bool displayed_help(int argc, const char* argv[]) { fprintf(stderr, "Usage:\n"); fprintf(stderr, "line_sender_c_example: [HOST [PORT]]\n"); - fprintf(stderr, " HOST: ILP host (defaults to \"localhost\").\n"); + fprintf( + stderr, " HOST: ILP host (defaults to \"localhost\").\n"); fprintf(stderr, " PORT: ILP port (defaults to \"9009\").\n"); return true; } diff --git a/examples/line_sender_c_example_array_byte_strides.c b/examples/line_sender_c_example_array_byte_strides.c index 8fc2da30..3d935e0c 100644 --- a/examples/line_sender_c_example_array_byte_strides.c +++ b/examples/line_sender_c_example_array_byte_strides.c @@ -12,7 +12,8 @@ static bool example(const char* host, const char* port) line_sender_error* err = NULL; line_sender* sender = NULL; line_sender_buffer* buffer = NULL; - char* conf_str = concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); + char* conf_str = + concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); if (!conf_str) { fprintf(stderr, "Could not concatenate configuration string.\n"); @@ -34,7 +35,8 @@ static bool example(const char* host, const char* port) buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); - line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("market_orders_byte_strides"); + line_sender_table_name table_name = + QDB_TABLE_NAME_LITERAL("market_orders_byte_strides"); line_sender_column_name symbol_col = QDB_COLUMN_NAME_LITERAL("symbol"); line_sender_column_name book_col = QDB_COLUMN_NAME_LITERAL("order_book"); diff --git a/examples/line_sender_c_example_array_c_major.c b/examples/line_sender_c_example_array_c_major.c index e9ce052e..50aa6dc7 100644 --- a/examples/line_sender_c_example_array_c_major.c +++ b/examples/line_sender_c_example_array_c_major.c @@ -12,7 +12,8 @@ static bool example(const char* host, const char* port) line_sender_error* err = NULL; line_sender* sender = NULL; line_sender_buffer* buffer = NULL; - char* conf_str = concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); + char* conf_str = + concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); if (!conf_str) { fprintf(stderr, "Could not concatenate configuration string.\n"); @@ -34,7 +35,8 @@ static bool example(const char* host, const char* port) buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); - line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("market_orders_c_major"); + line_sender_table_name table_name = + QDB_TABLE_NAME_LITERAL("market_orders_c_major"); line_sender_column_name symbol_col = QDB_COLUMN_NAME_LITERAL("symbol"); line_sender_column_name book_col = QDB_COLUMN_NAME_LITERAL("order_book"); diff --git a/examples/line_sender_c_example_array_elem_strides.c b/examples/line_sender_c_example_array_elem_strides.c index 96d02f68..fcbc041d 100644 --- a/examples/line_sender_c_example_array_elem_strides.c +++ b/examples/line_sender_c_example_array_elem_strides.c @@ -12,7 +12,8 @@ static bool example(const char* host, const char* port) line_sender_error* err = NULL; line_sender* sender = NULL; line_sender_buffer* buffer = NULL; - char* conf_str = concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); + char* conf_str = + concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); if (!conf_str) { fprintf(stderr, "Could not concatenate configuration string.\n"); @@ -34,7 +35,8 @@ static bool example(const char* host, const char* port) buffer = line_sender_buffer_new_for_sender(sender); line_sender_buffer_reserve(buffer, 64 * 1024); - line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("market_orders_elem_strides"); + line_sender_table_name table_name = + QDB_TABLE_NAME_LITERAL("market_orders_elem_strides"); line_sender_column_name symbol_col = QDB_COLUMN_NAME_LITERAL("symbol"); line_sender_column_name book_col = QDB_COLUMN_NAME_LITERAL("order_book"); diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index 5f4fb8c8..0be793df 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -10,18 +10,24 @@ static bool example(const char* host, const char* port) line_sender* sender = NULL; line_sender_buffer* buffer = NULL; char* conf_str = concat( - "tcp::addr=", host, ":", port, ";" + "tcp::addr=", + host, + ":", + port, + ";" "protocol_version=2;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" "token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac;"); - if (!conf_str) { + if (!conf_str) + { fprintf(stderr, "Could not concatenate configuration string.\n"); return false; } - line_sender_utf8 conf_str_utf8 = { 0, NULL }; - if (!line_sender_utf8_init(&conf_str_utf8, strlen(conf_str), conf_str, &err)) + line_sender_utf8 conf_str_utf8 = {0, NULL}; + if (!line_sender_utf8_init( + &conf_str_utf8, strlen(conf_str), conf_str, &err)) goto on_error; sender = line_sender_from_conf(conf_str_utf8, &err); @@ -32,7 +38,7 @@ static bool example(const char* host, const char* port) conf_str = NULL; buffer = line_sender_buffer_new_for_sender(sender); - line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. + line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid @@ -43,7 +49,6 @@ static bool example(const char* host, const char* port) line_sender_column_name price_name = QDB_COLUMN_NAME_LITERAL("price"); line_sender_column_name amount_name = QDB_COLUMN_NAME_LITERAL("amount"); - if (!line_sender_buffer_table(buffer, table_name, &err)) goto on_error; @@ -80,7 +85,7 @@ static bool example(const char* host, const char* port) return true; -on_error: ; +on_error:; size_t err_len = 0; const char* err_msg = line_sender_error_msg(err, &err_len); fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); @@ -100,7 +105,8 @@ static bool displayed_help(int argc, const char* argv[]) { fprintf(stderr, "Usage:\n"); fprintf(stderr, "line_sender_c_example_auth: [HOST [PORT]]\n"); - fprintf(stderr, " HOST: ILP host (defaults to \"localhost\").\n"); + fprintf( + stderr, " HOST: ILP host (defaults to \"localhost\").\n"); fprintf(stderr, " PORT: ILP port (defaults to \"9009\").\n"); return true; } diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index 4594afe6..d7613d29 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -10,18 +10,24 @@ static bool example(const char* host, const char* port) line_sender* sender = NULL; line_sender_buffer* buffer = NULL; char* conf_str = concat( - "tcps::addr=", host, ":", port, ";" + "tcps::addr=", + host, + ":", + port, + ";" "protocol_version=2;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" "token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac;"); - if (!conf_str) { + if (!conf_str) + { fprintf(stderr, "Could not concatenate configuration string.\n"); return false; } - line_sender_utf8 conf_str_utf8 = { 0, NULL }; - if (!line_sender_utf8_init(&conf_str_utf8, strlen(conf_str), conf_str, &err)) + line_sender_utf8 conf_str_utf8 = {0, NULL}; + if (!line_sender_utf8_init( + &conf_str_utf8, strlen(conf_str), conf_str, &err)) goto on_error; sender = line_sender_from_conf(conf_str_utf8, &err); @@ -32,18 +38,18 @@ static bool example(const char* host, const char* port) conf_str = NULL; buffer = line_sender_buffer_new_for_sender(sender); - line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. + line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid // re-validating the same strings over and over again. - line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_trades_auth_tls"); + line_sender_table_name table_name = + QDB_TABLE_NAME_LITERAL("c_trades_auth_tls"); line_sender_column_name symbol_name = QDB_COLUMN_NAME_LITERAL("symbol"); line_sender_column_name side_name = QDB_COLUMN_NAME_LITERAL("side"); line_sender_column_name price_name = QDB_COLUMN_NAME_LITERAL("price"); line_sender_column_name amount_name = QDB_COLUMN_NAME_LITERAL("amount"); - if (!line_sender_buffer_table(buffer, table_name, &err)) goto on_error; @@ -80,7 +86,7 @@ static bool example(const char* host, const char* port) return true; -on_error: ; +on_error:; size_t err_len = 0; const char* err_msg = line_sender_error_msg(err, &err_len); fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); @@ -100,7 +106,8 @@ static bool displayed_help(int argc, const char* argv[]) { fprintf(stderr, "Usage:\n"); fprintf(stderr, "line_sender_c_example_auth_tls: [HOST [PORT]]\n"); - fprintf(stderr, " HOST: ILP host (defaults to \"localhost\").\n"); + fprintf( + stderr, " HOST: ILP host (defaults to \"localhost\").\n"); fprintf(stderr, " PORT: ILP port (defaults to \"9009\").\n"); return true; } diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c index 122d0853..757071d0 100644 --- a/examples/line_sender_c_example_from_conf.c +++ b/examples/line_sender_c_example_from_conf.c @@ -8,25 +8,25 @@ int main(int argc, const char* argv[]) line_sender_error* err = NULL; line_sender_buffer* buffer = NULL; - line_sender_utf8 conf = QDB_UTF8_LITERAL( - "tcp::addr=localhost:9009;protocol_version=2;"); + line_sender_utf8 conf = + QDB_UTF8_LITERAL("tcp::addr=localhost:9009;protocol_version=2;"); line_sender* sender = line_sender_from_conf(conf, &err); if (!sender) goto on_error; buffer = line_sender_buffer_new_for_sender(sender); - line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. + line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid // re-validating the same strings over and over again. - line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_trades_from_conf"); + line_sender_table_name table_name = + QDB_TABLE_NAME_LITERAL("c_trades_from_conf"); line_sender_column_name symbol_name = QDB_COLUMN_NAME_LITERAL("symbol"); line_sender_column_name side_name = QDB_COLUMN_NAME_LITERAL("side"); line_sender_column_name price_name = QDB_COLUMN_NAME_LITERAL("price"); line_sender_column_name amount_name = QDB_COLUMN_NAME_LITERAL("amount"); - if (!line_sender_buffer_table(buffer, table_name, &err)) goto on_error; @@ -63,7 +63,7 @@ int main(int argc, const char* argv[]) return 0; -on_error: ; +on_error:; size_t err_len = 0; const char* err_msg = line_sender_error_msg(err, &err_len); fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); diff --git a/examples/line_sender_c_example_from_env.c b/examples/line_sender_c_example_from_env.c index 75904bc3..823c5928 100644 --- a/examples/line_sender_c_example_from_env.c +++ b/examples/line_sender_c_example_from_env.c @@ -14,18 +14,18 @@ int main(int argc, const char* argv[]) goto on_error; buffer = line_sender_buffer_new_for_sender(sender); - line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. + line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid // re-validating the same strings over and over again. - line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_trades_from_env"); + line_sender_table_name table_name = + QDB_TABLE_NAME_LITERAL("c_trades_from_env"); line_sender_column_name symbol_name = QDB_COLUMN_NAME_LITERAL("symbol"); line_sender_column_name side_name = QDB_COLUMN_NAME_LITERAL("side"); line_sender_column_name price_name = QDB_COLUMN_NAME_LITERAL("price"); line_sender_column_name amount_name = QDB_COLUMN_NAME_LITERAL("amount"); - if (!line_sender_buffer_table(buffer, table_name, &err)) goto on_error; @@ -62,7 +62,7 @@ int main(int argc, const char* argv[]) return 0; -on_error: ; +on_error:; size_t err_len = 0; const char* err_msg = line_sender_error_msg(err, &err_len); fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index 09a7fe93..5b9a82cb 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -12,12 +12,14 @@ static bool example(const char* host, const char* port) // Use `https` to enable TLS. // Use `username=...;password=...;` or `token=...` for authentication. char* conf_str = concat("http::addr=", host, ":", port, ";"); - if (!conf_str) { + if (!conf_str) + { fprintf(stderr, "Could not concatenate configuration string.\n"); return false; } - line_sender_utf8 conf_str_utf8 = { 0, NULL }; - if (!line_sender_utf8_init(&conf_str_utf8, strlen(conf_str), conf_str, &err)) + line_sender_utf8 conf_str_utf8 = {0, NULL}; + if (!line_sender_utf8_init( + &conf_str_utf8, strlen(conf_str), conf_str, &err)) goto on_error; sender = line_sender_from_conf(conf_str_utf8, &err); @@ -36,7 +38,6 @@ static bool example(const char* host, const char* port) line_sender_column_name price_name = QDB_COLUMN_NAME_LITERAL("price"); line_sender_column_name amount_name = QDB_COLUMN_NAME_LITERAL("amount"); - if (!line_sender_buffer_table(buffer, table_name, &err)) goto on_error; @@ -73,7 +74,7 @@ static bool example(const char* host, const char* port) return true; -on_error: ; +on_error:; size_t err_len = 0; const char* err_msg = line_sender_error_msg(err, &err_len); fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); @@ -93,7 +94,8 @@ static bool displayed_help(int argc, const char* argv[]) { fprintf(stderr, "Usage:\n"); fprintf(stderr, "line_sender_c_example: [HOST [PORT]]\n"); - fprintf(stderr, " HOST: ILP host (defaults to \"localhost\").\n"); + fprintf( + stderr, " HOST: ILP host (defaults to \"localhost\").\n"); fprintf(stderr, " PORT: HTTP port (defaults to \"9000\").\n"); return true; } diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index 058ee223..a5657ede 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -10,19 +10,27 @@ static bool example(const char* ca_path, const char* host, const char* port) line_sender* sender = NULL; line_sender_buffer* buffer = NULL; char* conf_str = concat( - "tcps::addr=", host, ":", port, ";", + "tcps::addr=", + host, + ":", + port, + ";", "protocol_version=2;" - "tls_roots=", ca_path, ";", + "tls_roots=", + ca_path, + ";", "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" "token_y=Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac;"); - if (!conf_str) { + if (!conf_str) + { fprintf(stderr, "Could not concatenate configuration string.\n"); return false; } - line_sender_utf8 conf_str_utf8 = { 0, NULL }; - if (!line_sender_utf8_init(&conf_str_utf8, strlen(conf_str), conf_str, &err)) + line_sender_utf8 conf_str_utf8 = {0, NULL}; + if (!line_sender_utf8_init( + &conf_str_utf8, strlen(conf_str), conf_str, &err)) goto on_error; sender = line_sender_from_conf(conf_str_utf8, &err); @@ -33,18 +41,18 @@ static bool example(const char* ca_path, const char* host, const char* port) conf_str = NULL; buffer = line_sender_buffer_new_for_sender(sender); - line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. + line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid // re-validating the same strings over and over again. - line_sender_table_name table_name = QDB_TABLE_NAME_LITERAL("c_trades_tls_ca"); + line_sender_table_name table_name = + QDB_TABLE_NAME_LITERAL("c_trades_tls_ca"); line_sender_column_name symbol_name = QDB_COLUMN_NAME_LITERAL("symbol"); line_sender_column_name side_name = QDB_COLUMN_NAME_LITERAL("side"); line_sender_column_name price_name = QDB_COLUMN_NAME_LITERAL("price"); line_sender_column_name amount_name = QDB_COLUMN_NAME_LITERAL("amount"); - if (!line_sender_buffer_table(buffer, table_name, &err)) goto on_error; @@ -81,7 +89,7 @@ static bool example(const char* ca_path, const char* host, const char* port) return true; -on_error: ; +on_error:; size_t err_len = 0; const char* err_msg = line_sender_error_msg(err, &err_len); fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); @@ -100,9 +108,12 @@ static bool displayed_help(int argc, const char* argv[]) if ((strncmp(arg, "-h", 2) == 0) || (strncmp(arg, "--help", 6) == 0)) { fprintf(stderr, "Usage:\n"); - fprintf(stderr, "line_sender_c_example_tls_ca: CA_PATH [HOST [PORT]]\n"); + fprintf( + stderr, + "line_sender_c_example_tls_ca: CA_PATH [HOST [PORT]]\n"); fprintf(stderr, " CA_PATH: Certificate authority pem file.\n"); - fprintf(stderr, " HOST: ILP host (defaults to \"localhost\").\n"); + fprintf( + stderr, " HOST: ILP host (defaults to \"localhost\").\n"); fprintf(stderr, " PORT: ILP port (defaults to \"9009\").\n"); return true; } diff --git a/examples/line_sender_cpp_example_array_byte_strides.cpp b/examples/line_sender_cpp_example_array_byte_strides.cpp index ce1d11c6..e2ee6c62 100644 --- a/examples/line_sender_cpp_example_array_byte_strides.cpp +++ b/examples/line_sender_cpp_example_array_byte_strides.cpp @@ -36,10 +36,14 @@ static bool array_example(std::string_view host, std::string_view port) 48121.5, 4.3}; - questdb::ingress::array::strided_view< - double, - questdb::ingress::array::strides_mode::bytes> - book_data{rank, shape.data(), strides.data(), arr_data.data(), arr_data.size()}; + questdb::ingress::array:: + strided_view + book_data{ + rank, + shape.data(), + strides.data(), + arr_data.data(), + arr_data.size()}; questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) diff --git a/examples/line_sender_cpp_example_array_c_major.cpp b/examples/line_sender_cpp_example_array_c_major.cpp index 637fc3da..baced0ae 100644 --- a/examples/line_sender_cpp_example_array_c_major.cpp +++ b/examples/line_sender_cpp_example_array_c_major.cpp @@ -35,8 +35,8 @@ static bool array_example(std::string_view host, std::string_view port) 48121.5, 4.3}; - questdb::ingress::array::row_major_view - book_data{rank, shape.data(), arr_data.data(), arr_data.size()}; + questdb::ingress::array::row_major_view book_data{ + rank, shape.data(), arr_data.data(), arr_data.size()}; questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) diff --git a/examples/line_sender_cpp_example_array_custom.cpp b/examples/line_sender_cpp_example_array_custom.cpp index 17cadf38..5009eabc 100644 --- a/examples/line_sender_cpp_example_array_custom.cpp +++ b/examples/line_sender_cpp_example_array_custom.cpp @@ -7,42 +7,71 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; -struct ViewHolder { +struct ViewHolder +{ std::array shape; std::array strides; const double* data; size_t size; - questdb::ingress::array::strided_view view() const { + questdb::ingress::array:: + strided_view + view() const + { return {2, shape.data(), strides.data(), data, size}; } }; -namespace custom_array { -class Matrix { +namespace custom_array +{ +class Matrix +{ public: Matrix(size_t rows, size_t cols) - : _rows{rows}, - _cols{cols}, - _data(rows * cols, std::numeric_limits::quiet_NaN()), - _row_stride{cols}, - _col_stride{1} {} + : _rows{rows} + , _cols{cols} + , _data(rows * cols, std::numeric_limits::quiet_NaN()) + , _row_stride{cols} + , _col_stride{1} + { + } - size_t rows() const { return _rows; } - size_t cols() const { return _cols; } - size_t row_stride() const { return _row_stride; } - size_t col_stride() const { return _col_stride; } - const double* data() const { return _data.data(); } - size_t size() const { return _data.size(); } + size_t rows() const + { + return _rows; + } + size_t cols() const + { + return _cols; + } + size_t row_stride() const + { + return _row_stride; + } + size_t col_stride() const + { + return _col_stride; + } + const double* data() const + { + return _data.data(); + } + size_t size() const + { + return _data.size(); + } - double get(size_t row, size_t col) const { + double get(size_t row, size_t col) const + { return _data[index(row, col)]; } - void set(size_t row, size_t col, double value) { + void set(size_t row, size_t col, double value) + { _data[index(row, col)] = value; } - void transpose() { + void transpose() + { std::swap(_rows, _cols); std::swap(_row_stride, _col_stride); } @@ -54,8 +83,10 @@ class Matrix { size_t _row_stride; size_t _col_stride; - size_t index(size_t row, size_t col) const { - if (row >= _rows || col >= _cols) { + size_t index(size_t row, size_t col) const + { + if (row >= _rows || col >= _cols) + { throw std::out_of_range("Matrix indices out of bounds"); } return row * _row_stride + col * _col_stride; @@ -65,28 +96,34 @@ class Matrix { // Customization point for QuestDB array API (discovered via König lookup) // If you need to support a 3rd party type, put this function in the namespace // of the type in question or in the `questdb::ingress::array` namespace -inline auto to_array_view_state_impl(const Matrix& m) { +inline auto to_array_view_state_impl(const Matrix& m) +{ return ViewHolder{ {static_cast(m.rows()), static_cast(m.cols())}, - {static_cast(m.row_stride()), static_cast(m.col_stride())}, + {static_cast(m.row_stride()), + static_cast(m.col_stride())}, m.data(), - m.size() - }; + m.size()}; } } // namespace custom_array static bool array_example(std::string_view host, std::string_view port) { using custom_array::Matrix; - try { + try + { auto sender = questdb::ingress::line_sender::from_conf( "http::addr=" + std::string{host} + ":" + std::string{port} + ";"); const auto table_name = "cpp_matrix_demo"_tn; const auto arr_col = "arr"_cn; Matrix m(2, 3); - m.set(0, 0, 1.1); m.set(0, 1, 2.2); m.set(0, 2, 3.3); - m.set(1, 0, 4.4); m.set(1, 1, 5.5); m.set(1, 2, 6.6); + m.set(0, 0, 1.1); + m.set(0, 1, 2.2); + m.set(0, 2, 3.3); + m.set(1, 0, 4.4); + m.set(1, 1, 5.5); + m.set(1, 2, 6.6); questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) @@ -102,7 +139,9 @@ static bool array_example(std::string_view host, std::string_view port) .at(questdb::ingress::timestamp_nanos::now()); sender.flush(buffer); return true; - } catch (const questdb::ingress::line_sender_error& err) { + } + catch (const questdb::ingress::line_sender_error& err) + { std::cerr << "[ERROR] " << err.what() << std::endl; return false; } diff --git a/examples/line_sender_cpp_example_array_elem_strides.cpp b/examples/line_sender_cpp_example_array_elem_strides.cpp index 688bcf2a..8418043e 100644 --- a/examples/line_sender_cpp_example_array_elem_strides.cpp +++ b/examples/line_sender_cpp_example_array_elem_strides.cpp @@ -38,7 +38,12 @@ static bool array_example(std::string_view host, std::string_view port) questdb::ingress::array::strided_view< double, questdb::ingress::array::strides_mode::elements> - book_data{3, shape.data(), strides.data(), arr_data.data(), arr_data.size()}; + book_data{ + 3, + shape.data(), + strides.data(), + arr_data.data(), + arr_data.size()}; questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); buffer.table(table_name) From 415cf8629473da84bff0f39264504f8f01b25419 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 19 Jun 2025 16:40:27 +0100 Subject: [PATCH 32/46] updated example sections for C and C++ --- doc/C.md | 23 +++++++++++++++++++---- doc/CPP.md | 24 ++++++++++++++++++++---- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/doc/C.md b/doc/C.md index 4f6b3e56..7956c96c 100644 --- a/doc/C.md +++ b/doc/C.md @@ -9,10 +9,25 @@ ## Complete Examples -* [Basic example in C](../examples/line_sender_c_example.c). -* [With authentication](../examples/line_sender_c_example_auth.c). -* [With authentication and TLS](../examples/line_sender_c_example_auth_tls.c). -* [Custom certificate authority file](../examples/line_sender_c_example_tls_ca.c). +**Basic Usage** +- [Basic example in C](../examples/line_sender_c_example.c) + +**Authentication & Security** +- [With authentication](../examples/line_sender_c_example_auth.c) +- [With authentication and TLS](../examples/line_sender_c_example_auth_tls.c) +- [Custom certificate authority file](../examples/line_sender_c_example_tls_ca.c) + +**Configuration** +- [Load configuration from file](../examples/line_sender_c_example_from_conf.c) +- [Load configuration from environment](../examples/line_sender_c_example_from_env.c) + +**HTTP** +- [Example using HTTP](../examples/line_sender_c_example_http.c) + +**Array Handling** +- [Array with byte strides](../examples/line_sender_c_example_array_byte_strides.c) +- [Array with element strides](../examples/line_sender_c_example_array_elem_strides.c) +- [Array in C-major order](../examples/line_sender_c_example_array_c_major.c) ## API Overview diff --git a/doc/CPP.md b/doc/CPP.md index 72e18794..9f2b1bf8 100644 --- a/doc/CPP.md +++ b/doc/CPP.md @@ -9,10 +9,26 @@ ## Complete Examples -* [Basic example in C++](../examples/line_sender_cpp_example.cpp). -* [With authentication](../examples/line_sender_cpp_example_auth.cpp). -* [With authentication and TLS](../examples/line_sender_cpp_example_auth_tls.cpp). -* [Custom certificate authority file](../examples/line_sender_c_example_tls_ca.c). +**Basic Usage** +- [Basic example in C++](../examples/line_sender_cpp_example.cpp) + +**Authentication & Security** +- [With authentication](../examples/line_sender_cpp_example_auth.cpp) +- [With authentication and TLS](../examples/line_sender_cpp_example_auth_tls.cpp) +- [Custom certificate authority file](../examples/line_sender_cpp_example_tls_ca.cpp) + +**Configuration** +- [Load configuration from file](../examples/line_sender_cpp_example_from_conf.cpp) +- [Load configuration from environment](../examples/line_sender_cpp_example_from_env.cpp) + +**HTTP** +- [Example using HTTP](../examples/line_sender_cpp_example_http.cpp) + +**Array Handling** +- [Array with byte strides](../examples/line_sender_cpp_example_array_byte_strides.cpp) +- [Array with element strides](../examples/line_sender_cpp_example_array_elem_strides.cpp) +- [Array in C-major order](../examples/line_sender_cpp_example_array_c_major.cpp) +- [Custom array type integration](../examples/line_sender_cpp_example_array_custom.cpp) ## API Overview From 47813374f1f7e6e193d98251d43b0138fde93674 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 19 Jun 2025 16:56:06 +0100 Subject: [PATCH 33/46] Fixed questdb-rs docs.rs not building --- questdb-rs/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index ff7ec416..dca30c98 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -11,7 +11,7 @@ categories = ["database"] authors = ["Adam Cimarosti "] [package.metadata.docs.rs] -all-features = true +features = ["almost-all-features"] [lib] name = "questdb" From 268ce5d0530a38877627460a924a672a1d28b51a Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 19 Jun 2025 18:24:57 +0100 Subject: [PATCH 34/46] examples summary for questdb-rs --- .bumpversion.cfg | 4 ++++ questdb-rs/README.md | 52 +++++++++++++++++++++++++++----------------- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c2feba7f..34026da6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -19,6 +19,10 @@ replace = version = "{new_version}" search = questdb-rs/{current_version}/ replace = questdb-rs/{new_version}/ +[bumpversion:file:questdb-rs/README.md] +search = https://github.com/questdb/c-questdb-client/tree/{current_version}/ +replace = https://github.com/questdb/c-questdb-client/tree/{new_version}/ + [bumpversion:file:questdb-rs-ffi/Cargo.toml] search = version = "{current_version}" replace = version = "{new_version}" diff --git a/questdb-rs/README.md b/questdb-rs/README.md index 292fb463..2795d62b 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -57,6 +57,12 @@ fn main() -> Result<()> { .symbol("side", "sell")? .column_f64("price", 2615.54)? .column_f64("amount", 0.00044)? + // Array ingestion (QuestDB 8.4.0+): + // 1. Store a price history as a Rust slice (e.g., last 3 trade prices) + .column_arr("price_history", &[2615.54, 2615.10, 2614.80][..])? + // 2. Store a volatility vector using ndarray (requires the `ndarray` feature) + // (e.g., 3-day rolling volatility values) + .column_arr("volatility", &ndarray::arr1(&[0.012, 0.011, 0.013]).view())? .at(TimestampNanos::now())?; sender.flush(&mut buffer)?; Ok(()) @@ -68,35 +74,41 @@ fn main() -> Result<()> { Most of the client documentation is on the [`ingress`](https://docs.rs/questdb-rs/5.0.0-rc1/questdb/ingress/) module page. -## Crate features +## Examples -This Rust crate supports a number of optional features, in most cases linked -to additional library dependencies. +A selection of usage examples is available in the [examples directory](https://github.com/questdb/c-questdb-client/tree/5.0.0-rc1/questdb-rs/examples): -For example, if you want to work with Chrono timestamps, use: +| Example | Description | +|---------|-------------| +| [`basic.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0-rc1/questdb-rs/examples/basic.rs) | Minimal TCP ingestion example; shows basic row and array ingestion. | +| [`auth.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0-rc1/questdb-rs/examples/auth.rs) | Adds authentication (user/password, token) to basic ingestion. | +| [`auth_tls.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0-rc1/questdb-rs/examples/auth_tls.rs) | Like `auth.rs`, but uses TLS for encrypted TCP connections. | +| [`from_conf.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0-rc1/questdb-rs/examples/from_conf.rs) | Configures client via connection string instead of builder pattern. | +| [`from_env.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0-rc1/questdb-rs/examples/from_env.rs) | Reads config from `QDB_CLIENT_CONF` environment variable. | +| [`http.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0-rc1/questdb-rs/examples/http.rs) | Uses HTTP transport and demonstrates array ingestion with `ndarray`. | +| [`protocol_version.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0-rc1/questdb-rs/examples/protocol_version.rs) | Shows protocol version selection and feature differences (e.g. arrays). | -```bash -cargo add questdb-rs --features chrono_timestamp -``` +## Crate features + +The crate provides several optional features to enable additional functionality. You can enable features using Cargo's `--features` flag or in your `Cargo.toml`. -### Default-enabled features +### Default features -* `ilp-over-http`: Enables ILP/HTTP support via the `ureq` crate. -* `tls-webpki-certs`: Supports using the `webpki-roots` crate for TLS - certificate verification. +- **ilp-over-http**: Enables ILP/HTTP support via the `ureq` crate for sending data over HTTP. +- **tls-webpki-certs**: Uses the `webpki-roots` crate to validate TLS certificates. +- **ring-crypto**: Uses the `ring` crate as the cryptography backend for TLS (default crypto backend). ### Optional features -These features are opt-in: +- **chrono_timestamp**: Allows specifying timestamps as `chrono::DateTime` objects. Requires the `chrono` crate. +- **tls-native-certs**: Uses OS-provided root TLS certificates for secure connections (via `rustls-native-certs`). +- **insecure-skip-verify**: Allows skipping verification of insecure certificates (not recommended for production). +- **ndarray**: Enables integration with the `ndarray` crate for working with n-dimensional arrays. Without this feature, you can still send slices or implement custom array types via the `NdArrayView` trait. +- **aws-lc-crypto**: Uses `aws-lc-rs` as the cryptography backend for TLS. Mutually exclusive with `ring-crypto`. + +- **almost-all-features**: Convenience feature for development and testing. Enables most features except mutually exclusive crypto backends. -* `chrono_timestamp`: Allows specifying timestamps as `chrono::Datetime` objects. -* `tls-native-certs`: Supports validating TLS certificates against the OS's - certificates store. -* `insecure-skip-verify`: Allows skipping server certificate validation in TLS - (this compromises security). -* `ndarray`: Enables integration with the `ndarray` crate for working with - n-dimensional arrays. Without this feature, you can still send slices, - or integrate custom array types via the `NdArrayView` trait. +> See the `Cargo.toml` for the full list and details on feature interactions. ## C, C++ and Python APIs From e03c2c231a6bf7fcac531b5e4b1ba03b11a607b4 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 19 Jun 2025 18:26:20 +0100 Subject: [PATCH 35/46] dummy test version --- .bumpversion.cfg | 2 +- doc/SECURITY.md | 4 ++-- include/questdb/ingress/line_sender.hpp | 2 +- questdb-rs-ffi/Cargo.lock | 4 ++-- questdb-rs-ffi/Cargo.toml | 2 +- questdb-rs/Cargo.toml | 2 +- questdb-rs/README.md | 18 +++++++++--------- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 34026da6..512dfbce 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 5.0.0-rc1 +current_version = 4.9.0 commit = False tag = False diff --git a/doc/SECURITY.md b/doc/SECURITY.md index 7b87fd0a..1e2d4ee8 100644 --- a/doc/SECURITY.md +++ b/doc/SECURITY.md @@ -35,7 +35,7 @@ A few important technical details on TLS: are managed centrally. For API usage: -* Rust: `SenderBuilder`'s [`auth`](https://docs.rs/questdb-rs/5.0.0-rc1/questdb/ingress/struct.SenderBuilder.html#method.auth) - and [`tls`](https://docs.rs/questdb-rs/5.0.0-rc1/questdb/ingress/struct.SenderBuilder.html#method.tls) methods. +* Rust: `SenderBuilder`'s [`auth`](https://docs.rs/questdb-rs/4.9.0/questdb/ingress/struct.SenderBuilder.html#method.auth) + and [`tls`](https://docs.rs/questdb-rs/4.9.0/questdb/ingress/struct.SenderBuilder.html#method.tls) methods. * C: [examples/line_sender_c_example_auth.c](../examples/line_sender_c_example_auth.c) * C++: [examples/line_sender_cpp_example_auth.cpp](../examples/line_sender_cpp_example_auth.cpp) diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index cd25fc33..741da502 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -1175,7 +1175,7 @@ class _user_agent static inline ::line_sender_utf8 name() { // Maintained by .bumpversion.cfg - static const char user_agent[] = "questdb/c++/5.0.0-rc1"; + static const char user_agent[] = "questdb/c++/4.9.0"; ::line_sender_utf8 utf8 = ::line_sender_utf8_assert(sizeof(user_agent) - 1, user_agent); return utf8; diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index d02045b8..497748f8 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -198,7 +198,7 @@ dependencies = [ [[package]] name = "questdb-rs" -version = "5.0.0-rc1" +version = "4.9.0" dependencies = [ "base64ct", "dns-lookup", @@ -224,7 +224,7 @@ dependencies = [ [[package]] name = "questdb-rs-ffi" -version = "5.0.0-rc1" +version = "4.9.0" dependencies = [ "libc", "questdb-confstr-ffi", diff --git a/questdb-rs-ffi/Cargo.toml b/questdb-rs-ffi/Cargo.toml index 1d461b5d..5e6e692b 100644 --- a/questdb-rs-ffi/Cargo.toml +++ b/questdb-rs-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "questdb-rs-ffi" -version = "5.0.0-rc1" +version = "4.9.0" edition = "2021" publish = false diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index dca30c98..721b2ab5 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "questdb-rs" -version = "5.0.0-rc1" +version = "4.9.0" edition = "2021" license = "Apache-2.0" description = "QuestDB Client Library for Rust" diff --git a/questdb-rs/README.md b/questdb-rs/README.md index 2795d62b..0d1e20ac 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -72,21 +72,21 @@ fn main() -> Result<()> { ## Docs Most of the client documentation is on the -[`ingress`](https://docs.rs/questdb-rs/5.0.0-rc1/questdb/ingress/) module page. +[`ingress`](https://docs.rs/questdb-rs/4.9.0/questdb/ingress/) module page. ## Examples -A selection of usage examples is available in the [examples directory](https://github.com/questdb/c-questdb-client/tree/5.0.0-rc1/questdb-rs/examples): +A selection of usage examples is available in the [examples directory](https://github.com/questdb/c-questdb-client/tree/4.9.0/questdb-rs/examples): | Example | Description | |---------|-------------| -| [`basic.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0-rc1/questdb-rs/examples/basic.rs) | Minimal TCP ingestion example; shows basic row and array ingestion. | -| [`auth.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0-rc1/questdb-rs/examples/auth.rs) | Adds authentication (user/password, token) to basic ingestion. | -| [`auth_tls.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0-rc1/questdb-rs/examples/auth_tls.rs) | Like `auth.rs`, but uses TLS for encrypted TCP connections. | -| [`from_conf.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0-rc1/questdb-rs/examples/from_conf.rs) | Configures client via connection string instead of builder pattern. | -| [`from_env.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0-rc1/questdb-rs/examples/from_env.rs) | Reads config from `QDB_CLIENT_CONF` environment variable. | -| [`http.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0-rc1/questdb-rs/examples/http.rs) | Uses HTTP transport and demonstrates array ingestion with `ndarray`. | -| [`protocol_version.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0-rc1/questdb-rs/examples/protocol_version.rs) | Shows protocol version selection and feature differences (e.g. arrays). | +| [`basic.rs`](https://github.com/questdb/c-questdb-client/blob/4.9.0/questdb-rs/examples/basic.rs) | Minimal TCP ingestion example; shows basic row and array ingestion. | +| [`auth.rs`](https://github.com/questdb/c-questdb-client/blob/4.9.0/questdb-rs/examples/auth.rs) | Adds authentication (user/password, token) to basic ingestion. | +| [`auth_tls.rs`](https://github.com/questdb/c-questdb-client/blob/4.9.0/questdb-rs/examples/auth_tls.rs) | Like `auth.rs`, but uses TLS for encrypted TCP connections. | +| [`from_conf.rs`](https://github.com/questdb/c-questdb-client/blob/4.9.0/questdb-rs/examples/from_conf.rs) | Configures client via connection string instead of builder pattern. | +| [`from_env.rs`](https://github.com/questdb/c-questdb-client/blob/4.9.0/questdb-rs/examples/from_env.rs) | Reads config from `QDB_CLIENT_CONF` environment variable. | +| [`http.rs`](https://github.com/questdb/c-questdb-client/blob/4.9.0/questdb-rs/examples/http.rs) | Uses HTTP transport and demonstrates array ingestion with `ndarray`. | +| [`protocol_version.rs`](https://github.com/questdb/c-questdb-client/blob/4.9.0/questdb-rs/examples/protocol_version.rs) | Shows protocol version selection and feature differences (e.g. arrays). | ## Crate features From 8d7b7700aecafbbd079b5ed354e8e66deef0e694 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 19 Jun 2025 18:40:33 +0100 Subject: [PATCH 36/46] upgraded to maintained bump-my-version tool --- .bumpversion.cfg | 32 -------------------------------- .bumpversion.toml | 39 +++++++++++++++++++++++++++++++++++++++ doc/DEV_NOTES.md | 21 --------------------- doc/RELEASING.md | 16 ++++++++-------- 4 files changed, 47 insertions(+), 61 deletions(-) delete mode 100644 .bumpversion.cfg create mode 100644 .bumpversion.toml diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 512dfbce..00000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,32 +0,0 @@ -[bumpversion] -current_version = 4.9.0 -commit = False -tag = False - -[bumpversion:file:CMakeLists.txt] -search = project(c-questdb-client VERSION {current_version}) -replace = project(c-questdb-client VERSION {new_version}) - -[bumpversion:file:doc/SECURITY.md] -search = questdb-rs/{current_version}/ -replace = questdb-rs/{new_version}/ - -[bumpversion:file:questdb-rs/Cargo.toml] -search = version = "{current_version}" -replace = version = "{new_version}" - -[bumpversion:file:./questdb-rs/README.md] -search = questdb-rs/{current_version}/ -replace = questdb-rs/{new_version}/ - -[bumpversion:file:questdb-rs/README.md] -search = https://github.com/questdb/c-questdb-client/tree/{current_version}/ -replace = https://github.com/questdb/c-questdb-client/tree/{new_version}/ - -[bumpversion:file:questdb-rs-ffi/Cargo.toml] -search = version = "{current_version}" -replace = version = "{new_version}" - -[bumpversion:file:include/questdb/ingress/line_sender.hpp] -search = questdb/c++/{current_version} -replace = questdb/c++/{new_version} diff --git a/.bumpversion.toml b/.bumpversion.toml new file mode 100644 index 00000000..c996f0a6 --- /dev/null +++ b/.bumpversion.toml @@ -0,0 +1,39 @@ +[tool.bumpversion] +current_version = "4.9.0" +commit = false +tag = false + +[[tool.bumpversion.files]] +filename = "CMakeLists.txt" +search = "project(c-questdb-client VERSION {current_version})" +replace = "project(c-questdb-client VERSION {new_version})" + +[[tool.bumpversion.files]] +filename = "doc/SECURITY.md" +search = "questdb-rs/{current_version}/" +replace = "questdb-rs/{new_version}/" + +[[tool.bumpversion.files]] +filename = "questdb-rs/Cargo.toml" +search = "version = \"{current_version}\"" +replace = "version = \"{new_version}\"" + +[[tool.bumpversion.files]] +filename = "./questdb-rs/README.md" +search = "questdb-rs/{current_version}/" +replace = "questdb-rs/{new_version}/" + +[[tool.bumpversion.files]] +filename = "questdb-rs/README.md" +search = "https://github.com/questdb/c-questdb-client/tree/{current_version}/" +replace = "https://github.com/questdb/c-questdb-client/tree/{new_version}/" + +[[tool.bumpversion.files]] +filename = "questdb-rs-ffi/Cargo.toml" +search = "version = \"{current_version}\"" +replace = "version = \"{new_version}\"" + +[[tool.bumpversion.files]] +filename = "include/questdb/ingress/line_sender.hpp" +search = "questdb/c++/{current_version}" +replace = "questdb/c++/{new_version}" \ No newline at end of file diff --git a/doc/DEV_NOTES.md b/doc/DEV_NOTES.md index 7dd5889e..e168b538 100644 --- a/doc/DEV_NOTES.md +++ b/doc/DEV_NOTES.md @@ -44,24 +44,3 @@ which also contains additional formatting and comments. This generated files should be not be checked in: * `include/questdb/ingress/line_sender.gen.h` * `cython/questdb/ingress/line_sender.pxd` - -## Updating version in the codebase before releasing - -* Ensure you have `python3` and `bump2version` installed (`python3 -m pip install bump2version`). - -```console -bump2version --config-file .bumpversion.cfg patch -``` - -Last argument argument: - * `patch` would bump from (for example) `0.1.0` to `0.1.1`. - * `minor` would bump from `0.1.0` to `0.2.0`. - * `major` would bump from `0.1.0` to `1.0.0`. - -* For more command line options, see: https://pypi.org/project/bump2version/ - -If you're editing the config file, a good set of arguments to debug issues is: - -``` -bump2version --dry-run --allow-dirty --verbose --config-file .bumpversion.cfg patch -``` diff --git a/doc/RELEASING.md b/doc/RELEASING.md index ec47c000..b16a897b 100644 --- a/doc/RELEASING.md +++ b/doc/RELEASING.md @@ -16,17 +16,17 @@ git switch -c vX.Y.Z ## 3. Update all the occurrences of the version in the repo -```bash -bump2version --config-file .bumpversion.cfg -``` +## Updating version in the codebase before releasing -The `increment` parameter can be: +* Ensure you have `uv` and `bump-my-version` installed: + * `curl -LsSf https://astral.sh/uv/install.sh | sh` : see https://docs.astral.sh/uv/getting-started/installation/ + * `uv tool install bump-my-version`: see https://github.com/callowayproject/bump-my-version. -- `patch` -- `minor` -- `major` +```console +bump-my-version replace --new-version NEW_VERSION +``` -Use the one appropriate to the version increment you're releasing. +If you're unsure, append `--dry-run` to preview changes. ## 4. Refresh `Cargo.lock` From 2586e55cc4659da541e3e43c1c7d9fe2da74e4a9 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 19 Jun 2025 18:41:41 +0100 Subject: [PATCH 37/46] fixed one that got missed.. which is why we have this stuff in the first place --- CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ca613404..109ce4d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,5 @@ cmake_minimum_required(VERSION 3.15.0) -project(c-questdb-client VERSION 5.0.0) -set(PROJECT_PRE_RELEASE "rc1") +project(c-questdb-client VERSION 4.9.0) set(CPACK_PROJECT_NAME ${PROJECT_NAME}) set(CPACK_PROJECT_VERSION ${PROJECT_VERSION}) From 91b7cf342b3e38a56f712b5a74da58503edf9e83 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 19 Jun 2025 18:44:14 +0100 Subject: [PATCH 38/46] fixed up .bumpversion.toml --- .bumpversion.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.bumpversion.toml b/.bumpversion.toml index c996f0a6..ef686416 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -28,6 +28,11 @@ filename = "questdb-rs/README.md" search = "https://github.com/questdb/c-questdb-client/tree/{current_version}/" replace = "https://github.com/questdb/c-questdb-client/tree/{new_version}/" +[[tool.bumpversion.files]] +filename = "questdb-rs/README.md" +search = "https://github.com/questdb/c-questdb-client/blob/{current_version}/questdb-rs/" +replace = "https://github.com/questdb/c-questdb-client/blob/{new_version}/questdb-rs/" + [[tool.bumpversion.files]] filename = "questdb-rs-ffi/Cargo.toml" search = "version = \"{current_version}\"" From 8bb4eec5d009425929380dc77dfca4735110fcde Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 19 Jun 2025 18:45:44 +0100 Subject: [PATCH 39/46] updated version to 5.0.0 --- CMakeLists.txt | 2 +- doc/SECURITY.md | 4 ++-- include/questdb/ingress/line_sender.hpp | 2 +- questdb-rs-ffi/Cargo.lock | 4 ++-- questdb-rs-ffi/Cargo.toml | 2 +- questdb-rs/Cargo.toml | 2 +- questdb-rs/README.md | 18 +++++++++--------- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 109ce4d9..52b1cabd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.15.0) -project(c-questdb-client VERSION 4.9.0) +project(c-questdb-client VERSION 5.0.0) set(CPACK_PROJECT_NAME ${PROJECT_NAME}) set(CPACK_PROJECT_VERSION ${PROJECT_VERSION}) diff --git a/doc/SECURITY.md b/doc/SECURITY.md index 1e2d4ee8..83b2e94b 100644 --- a/doc/SECURITY.md +++ b/doc/SECURITY.md @@ -35,7 +35,7 @@ A few important technical details on TLS: are managed centrally. For API usage: -* Rust: `SenderBuilder`'s [`auth`](https://docs.rs/questdb-rs/4.9.0/questdb/ingress/struct.SenderBuilder.html#method.auth) - and [`tls`](https://docs.rs/questdb-rs/4.9.0/questdb/ingress/struct.SenderBuilder.html#method.tls) methods. +* Rust: `SenderBuilder`'s [`auth`](https://docs.rs/questdb-rs/5.0.0/questdb/ingress/struct.SenderBuilder.html#method.auth) + and [`tls`](https://docs.rs/questdb-rs/5.0.0/questdb/ingress/struct.SenderBuilder.html#method.tls) methods. * C: [examples/line_sender_c_example_auth.c](../examples/line_sender_c_example_auth.c) * C++: [examples/line_sender_cpp_example_auth.cpp](../examples/line_sender_cpp_example_auth.cpp) diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 741da502..a9b95303 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -1175,7 +1175,7 @@ class _user_agent static inline ::line_sender_utf8 name() { // Maintained by .bumpversion.cfg - static const char user_agent[] = "questdb/c++/4.9.0"; + static const char user_agent[] = "questdb/c++/5.0.0"; ::line_sender_utf8 utf8 = ::line_sender_utf8_assert(sizeof(user_agent) - 1, user_agent); return utf8; diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index 497748f8..2d7d8799 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -198,7 +198,7 @@ dependencies = [ [[package]] name = "questdb-rs" -version = "4.9.0" +version = "5.0.0" dependencies = [ "base64ct", "dns-lookup", @@ -224,7 +224,7 @@ dependencies = [ [[package]] name = "questdb-rs-ffi" -version = "4.9.0" +version = "5.0.0" dependencies = [ "libc", "questdb-confstr-ffi", diff --git a/questdb-rs-ffi/Cargo.toml b/questdb-rs-ffi/Cargo.toml index 5e6e692b..353f6dc6 100644 --- a/questdb-rs-ffi/Cargo.toml +++ b/questdb-rs-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "questdb-rs-ffi" -version = "4.9.0" +version = "5.0.0" edition = "2021" publish = false diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 721b2ab5..94244dec 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "questdb-rs" -version = "4.9.0" +version = "5.0.0" edition = "2021" license = "Apache-2.0" description = "QuestDB Client Library for Rust" diff --git a/questdb-rs/README.md b/questdb-rs/README.md index 0d1e20ac..c8de2efd 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -72,21 +72,21 @@ fn main() -> Result<()> { ## Docs Most of the client documentation is on the -[`ingress`](https://docs.rs/questdb-rs/4.9.0/questdb/ingress/) module page. +[`ingress`](https://docs.rs/questdb-rs/5.0.0/questdb/ingress/) module page. ## Examples -A selection of usage examples is available in the [examples directory](https://github.com/questdb/c-questdb-client/tree/4.9.0/questdb-rs/examples): +A selection of usage examples is available in the [examples directory](https://github.com/questdb/c-questdb-client/tree/5.0.0/questdb-rs/examples): | Example | Description | |---------|-------------| -| [`basic.rs`](https://github.com/questdb/c-questdb-client/blob/4.9.0/questdb-rs/examples/basic.rs) | Minimal TCP ingestion example; shows basic row and array ingestion. | -| [`auth.rs`](https://github.com/questdb/c-questdb-client/blob/4.9.0/questdb-rs/examples/auth.rs) | Adds authentication (user/password, token) to basic ingestion. | -| [`auth_tls.rs`](https://github.com/questdb/c-questdb-client/blob/4.9.0/questdb-rs/examples/auth_tls.rs) | Like `auth.rs`, but uses TLS for encrypted TCP connections. | -| [`from_conf.rs`](https://github.com/questdb/c-questdb-client/blob/4.9.0/questdb-rs/examples/from_conf.rs) | Configures client via connection string instead of builder pattern. | -| [`from_env.rs`](https://github.com/questdb/c-questdb-client/blob/4.9.0/questdb-rs/examples/from_env.rs) | Reads config from `QDB_CLIENT_CONF` environment variable. | -| [`http.rs`](https://github.com/questdb/c-questdb-client/blob/4.9.0/questdb-rs/examples/http.rs) | Uses HTTP transport and demonstrates array ingestion with `ndarray`. | -| [`protocol_version.rs`](https://github.com/questdb/c-questdb-client/blob/4.9.0/questdb-rs/examples/protocol_version.rs) | Shows protocol version selection and feature differences (e.g. arrays). | +| [`basic.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0/questdb-rs/examples/basic.rs) | Minimal TCP ingestion example; shows basic row and array ingestion. | +| [`auth.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0/questdb-rs/examples/auth.rs) | Adds authentication (user/password, token) to basic ingestion. | +| [`auth_tls.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0/questdb-rs/examples/auth_tls.rs) | Like `auth.rs`, but uses TLS for encrypted TCP connections. | +| [`from_conf.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0/questdb-rs/examples/from_conf.rs) | Configures client via connection string instead of builder pattern. | +| [`from_env.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0/questdb-rs/examples/from_env.rs) | Reads config from `QDB_CLIENT_CONF` environment variable. | +| [`http.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0/questdb-rs/examples/http.rs) | Uses HTTP transport and demonstrates array ingestion with `ndarray`. | +| [`protocol_version.rs`](https://github.com/questdb/c-questdb-client/blob/5.0.0/questdb-rs/examples/protocol_version.rs) | Shows protocol version selection and feature differences (e.g. arrays). | ## Crate features From 8d6ce96dbbf46cdef7837b20d3378b2d1c170896 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 19 Jun 2025 21:07:30 +0100 Subject: [PATCH 40/46] fixed readme compile issue --- questdb-rs/README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/questdb-rs/README.md b/questdb-rs/README.md index c8de2efd..b5025852 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -40,7 +40,7 @@ cargo add questdb-rs Then you can try out this quick example, which connects to a QuestDB server running on your local machine: -```rust no_run +```rust ignore use questdb::{ Result, ingress::{ @@ -57,12 +57,10 @@ fn main() -> Result<()> { .symbol("side", "sell")? .column_f64("price", 2615.54)? .column_f64("amount", 0.00044)? - // Array ingestion (QuestDB 8.4.0+): - // 1. Store a price history as a Rust slice (e.g., last 3 trade prices) - .column_arr("price_history", &[2615.54, 2615.10, 2614.80][..])? - // 2. Store a volatility vector using ndarray (requires the `ndarray` feature) - // (e.g., 3-day rolling volatility values) - .column_arr("volatility", &ndarray::arr1(&[0.012, 0.011, 0.013]).view())? + + // Array ingestion (QuestDB 8.4.0+). Slices and ndarray supported through trait + .column_arr("price_history", &[2615.54f64, 2615.10, 2614.80])? + .column_arr("volatility", &ndarray::arr1(&[0.012f64, 0.011, 0.013]).view())? .at(TimestampNanos::now())?; sender.flush(&mut buffer)?; Ok(()) From 2c4360047b41b8a405fde56c84fe624eb4a9e942 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 19 Jun 2025 21:14:32 +0100 Subject: [PATCH 41/46] Comment issue spotted by Copilot AI --- questdb-rs/src/ingress/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 7b77ccc6..962d40dc 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -621,7 +621,7 @@ impl Buffer { /// This is equivalent to [`Sender::new_buffer`] when using the [`Sender::protocol_version`] /// and [`Sender::max_name_len`]. /// - /// For the default max name length limit (32), use [`Self::new`]. + /// For the default max name length limit (127), use [`Self::new`]. pub fn with_max_name_len(protocol_version: ProtocolVersion, max_name_len: usize) -> Self { Self { output: Vec::new(), From 5af7515a29bc5b612516474a83e1186c583a73b3 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Fri, 20 Jun 2025 10:53:32 +0100 Subject: [PATCH 42/46] .bumpversion.toml fixup --- .bumpversion.toml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index ef686416..37669f5c 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "4.9.0" +current_version = "5.0.0" commit = false tag = false @@ -41,4 +41,9 @@ replace = "version = \"{new_version}\"" [[tool.bumpversion.files]] filename = "include/questdb/ingress/line_sender.hpp" search = "questdb/c++/{current_version}" -replace = "questdb/c++/{new_version}" \ No newline at end of file +replace = "questdb/c++/{new_version}" + +[[tool.bumpversion.files]] +filename = ".bumpversion.toml" +search = "current_version = \"{current_version}\"" +replace = "current_version = \"{new_version}\"" From 80cb3f84ce35e8e34defee2596a1626eba65f915 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Thu, 26 Jun 2025 11:24:52 +0100 Subject: [PATCH 43/46] lower limit for MAX_ARRAY_BUFFER_SIZE --- questdb-rs/src/ingress/mod.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 962d40dc..b7298030 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -62,13 +62,7 @@ const MAX_NAME_LEN_DEFAULT: usize = 127; /// The maximum allowed dimensions for arrays. pub const MAX_ARRAY_DIMS: usize = 32; - -// TODO: We should probably agree on a significantly -// _smaller_ limit here, since there's no way -// we've ever tested anything that big. -// My gut feeling is that the maximum array buffer should be -// in the order of 100MB or so. -pub const MAX_ARRAY_BUFFER_SIZE: usize = i32::MAX as usize; +pub const MAX_ARRAY_BUFFER_SIZE: usize = 512 * 1024 * 1024; // 512MiB pub const MAX_ARRAY_DIM_LEN: usize = 0x0FFF_FFFF; // 1 << 28 - 1 /// The version of InfluxDB Line Protocol used to communicate with the server. From 860891b9e42d449caf837322aa677cf9b91fa3a5 Mon Sep 17 00:00:00 2001 From: amunra Date: Wed, 2 Jul 2025 12:22:28 +0100 Subject: [PATCH 44/46] cargo clippy --fix --- questdb-rs/build.rs | 23 +++++++++++------------ questdb-rs/src/gai.rs | 2 +- questdb-rs/src/ingress/http.rs | 4 ++-- questdb-rs/src/ingress/mod.rs | 6 +++--- questdb-rs/src/ingress/tests.rs | 4 ++-- questdb-rs/src/tests/mock.rs | 5 ++--- questdb-rs/src/tests/mod.rs | 7 +++---- questdb-rs/src/tests/ndarr.rs | 2 +- 8 files changed, 25 insertions(+), 28 deletions(-) diff --git a/questdb-rs/build.rs b/questdb-rs/build.rs index 98dc34ef..5ad8f336 100644 --- a/questdb-rs/build.rs +++ b/questdb-rs/build.rs @@ -162,7 +162,7 @@ pub mod json_tests { if expected.is_none() { writeln!(output, " || -> Result<()> {{")?; } - writeln!(output, "{} buffer", indent)?; + writeln!(output, "{indent} buffer")?; writeln!(output, "{} .table({:?})?", indent, spec.table)?; for symbol in spec.symbols.iter() { writeln!( @@ -195,14 +195,13 @@ pub mod json_tests { )?, } } - writeln!(output, "{} .at_now()?;", indent)?; + writeln!(output, "{indent} .at_now()?;")?; if let Some(expected) = expected { if let Some(ref base64) = expected.binary_base64 { writeln!(output, " if version != ProtocolVersion::V1 {{")?; writeln!( output, - " let exp = Base64::decode_vec(\"{}\").unwrap();", - base64 + " let exp = Base64::decode_vec(\"{base64}\").unwrap();" )?; writeln!( output, @@ -210,8 +209,8 @@ pub mod json_tests { )?; writeln!(output, " }} else {{")?; if let Some(ref line) = expected.line { - let exp_ln = format!("{}\n", line); - writeln!(output, " let exp = {:?};", exp_ln)?; + let exp_ln = format!("{line}\n"); + writeln!(output, " let exp = {exp_ln:?};")?; writeln!( output, " assert_eq!(buffer.as_bytes(), exp.as_bytes());" @@ -223,11 +222,11 @@ pub mod json_tests { .as_ref() .unwrap() .iter() - .map(|line| format!("{}\n", line)) + .map(|line| format!("{line}\n")) .collect(); writeln!(output, " let any = [")?; for line in any.iter() { - writeln!(output, " {:?},", line)?; + writeln!(output, " {line:?},")?; } writeln!(output, " ];")?; writeln!( @@ -237,8 +236,8 @@ pub mod json_tests { } writeln!(output, " }}")?; } else if let Some(ref line) = expected.line { - let exp_ln = format!("{}\n", line); - writeln!(output, " let exp = {:?};", exp_ln)?; + let exp_ln = format!("{line}\n"); + writeln!(output, " let exp = {exp_ln:?};")?; writeln!(output, " assert_eq!(buffer.as_bytes(), exp.as_bytes());")?; } else { let any: Vec = expected @@ -246,11 +245,11 @@ pub mod json_tests { .as_ref() .unwrap() .iter() - .map(|line| format!("{}\n", line)) + .map(|line| format!("{line}\n")) .collect(); writeln!(output, " let any = [")?; for line in any.iter() { - writeln!(output, " {:?},", line)?; + writeln!(output, " {line:?},")?; } writeln!(output, " ];")?; writeln!( diff --git a/questdb-rs/src/gai.rs b/questdb-rs/src/gai.rs index 145dac3e..ccb36651 100644 --- a/questdb-rs/src/gai.rs +++ b/questdb-rs/src/gai.rs @@ -75,7 +75,7 @@ pub(super) fn resolve_host_port(host: &str, port: &str) -> super::Result Error { description.push_str(", "); } description.push_str("line: "); - write!(description, "{}", line).unwrap(); + write!(description, "{line}").unwrap(); } description.push(']'); @@ -337,7 +337,7 @@ pub(super) fn parse_http_error(http_status_code: u16, response: Response) ); } else if [401, 403].contains(&http_status_code) { let description = match body_content { - Ok(msg) if !msg.is_empty() => format!(": {}", msg), + Ok(msg) if !msg.is_empty() => format!(": {msg}"), _ => "".to_string(), }; return error::fmt!( diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index b7298030..3fd124d9 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -357,7 +357,7 @@ enum Connection { impl Connection { fn send_key_id(&mut self, key_id: &str) -> Result<()> { - writeln!(self, "{}", key_id) + writeln!(self, "{key_id}") .map_err(|io_err| map_io_to_socket_err("Failed to send key_id: ", io_err))?; Ok(()) } @@ -2366,14 +2366,14 @@ impl SenderBuilder { let bind_addr = gai::resolve_host(host.as_str())?; sock.bind(&bind_addr).map_err(|io_err| { map_io_to_socket_err( - &format!("Could not bind to interface address {:?}: ", host), + &format!("Could not bind to interface address {host:?}: "), io_err, ) })?; } sock.connect(&addr).map_err(|io_err| { let host_port = format!("{}:{}", self.host.deref(), *self.port); - let prefix = format!("Could not connect to {:?}: ", host_port); + let prefix = format!("Could not connect to {host_port:?}: "); map_io_to_socket_err(&prefix, io_err) })?; diff --git a/questdb-rs/src/ingress/tests.rs b/questdb-rs/src/ingress/tests.rs index 69a8bd6d..0d896e2d 100644 --- a/questdb-rs/src/ingress/tests.rs +++ b/questdb-rs/src/ingress/tests.rs @@ -530,7 +530,7 @@ fn assert_specified_eq>( if let ConfigSetting::Specified(actual_value) = actual { assert_eq!(actual_value, &expected); } else { - panic!("Expected Specified({:?}), but got {:?}", expected, actual); + panic!("Expected Specified({expected:?}), but got {actual:?}"); } } @@ -542,7 +542,7 @@ fn assert_defaulted_eq>( if let ConfigSetting::Defaulted(actual_value) = actual { assert_eq!(actual_value, &expected); } else { - panic!("Expected Defaulted({:?}), but got {:?}", expected, actual); + panic!("Expected Defaulted({expected:?}), but got {actual:?}"); } } diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index d20d3456..d9fba08e 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -173,7 +173,7 @@ impl HttpResponse { pub fn as_string(&self) -> String { let mut s = format!("HTTP/1.1 {} {}\r\n", self.status_code, self.status_text); for (key, value) in &self.headers { - s.push_str(&format!("{}: {}\r\n", key, value)); + s.push_str(&format!("{key}: {value}\r\n")); } s.push_str("\r\n"); s.push_str(std::str::from_utf8(&self.body).unwrap()); @@ -395,8 +395,7 @@ impl MockServer { return Err(io::Error::new( io::ErrorKind::TimedOut, format!( - "{} timed out while waiting for data. Received so far: {}", - stage, so_far + "{stage} timed out while waiting for data. Received so far: {so_far}" ), )); } diff --git a/questdb-rs/src/tests/mod.rs b/questdb-rs/src/tests/mod.rs index c63a65d3..6bbf55b7 100644 --- a/questdb-rs/src/tests/mod.rs +++ b/questdb-rs/src/tests/mod.rs @@ -46,10 +46,9 @@ pub fn assert_err_contains( expected_msg_contained: &str, ) { match result { - Ok(_) => panic!( - "Expected error containing '{}', but got Ok({:?})", - expected_msg_contained, result - ), + Ok(_) => { + panic!("Expected error containing '{expected_msg_contained}', but got Ok({result:?})") + } Err(e) => { assert_eq!( e.code(), diff --git a/questdb-rs/src/tests/ndarr.rs b/questdb-rs/src/tests/ndarr.rs index 1e5e3bb9..b43f0437 100644 --- a/questdb-rs/src/tests/ndarr.rs +++ b/questdb-rs/src/tests/ndarr.rs @@ -62,7 +62,7 @@ impl TryFrom for ArrayColumnTypeTag { fn try_from(value: u8) -> Result { match value { 10 => Ok(ArrayColumnTypeTag::Double), - _ => Err(format!("Unsupported column type tag {} for arrays", value)), + _ => Err(format!("Unsupported column type tag {value} for arrays")), } } } From e9cace21970f607bca6393b218fd1a4dc12e0859 Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Wed, 2 Jul 2025 13:07:07 +0100 Subject: [PATCH 45/46] bumped named version supporting arrays from 8.4.0to 9.0.0 --- README.md | 2 +- examples/line_sender_c_example_array_byte_strides.c | 2 +- examples/line_sender_c_example_array_c_major.c | 2 +- examples/line_sender_c_example_array_elem_strides.c | 2 +- .../line_sender_cpp_example_array_byte_strides.cpp | 2 +- examples/line_sender_cpp_example_array_c_major.cpp | 2 +- .../line_sender_cpp_example_array_elem_strides.cpp | 2 +- include/questdb/ingress/line_sender.h | 10 +++++----- include/questdb/ingress/line_sender.hpp | 10 +++++----- questdb-rs-ffi/src/lib.rs | 10 +++++----- questdb-rs/README.md | 6 +++--- questdb-rs/examples/basic.rs | 2 +- questdb-rs/examples/http.rs | 2 +- questdb-rs/examples/protocol_version.rs | 2 +- questdb-rs/src/ingress/mod.md | 4 ++-- questdb-rs/src/ingress/mod.rs | 6 +++--- 16 files changed, 33 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 306ae876..d1563b5f 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ These protocol versions are supported over both HTTP and TCP. | Version | Description | Server Compatibility | | ------- | ------------------------------------------------------- | --------------------- | | **1** | Over HTTP it's compatible InfluxDB Line Protocol (ILP) | All QuestDB versions | -| **2** | 64-bit floats sent as binary, adds n-dimentional arrays | 8.4.0+ (2023-10-30) | +| **2** | 64-bit floats sent as binary, adds n-dimentional arrays | 9.0.0+ (2023-10-30) | ## Getting Started diff --git a/examples/line_sender_c_example_array_byte_strides.c b/examples/line_sender_c_example_array_byte_strides.c index 3d935e0c..efe65f33 100644 --- a/examples/line_sender_c_example_array_byte_strides.c +++ b/examples/line_sender_c_example_array_byte_strides.c @@ -5,7 +5,7 @@ #include "concat.h" /* - * QuestDB server version 8.4.0 or later is required for array support. + * QuestDB server version 9.0.0 or later is required for array support. */ static bool example(const char* host, const char* port) { diff --git a/examples/line_sender_c_example_array_c_major.c b/examples/line_sender_c_example_array_c_major.c index 50aa6dc7..798b13ba 100644 --- a/examples/line_sender_c_example_array_c_major.c +++ b/examples/line_sender_c_example_array_c_major.c @@ -5,7 +5,7 @@ #include "concat.h" /* - * QuestDB server version 8.4.0 or later is required for array support. + * QuestDB server version 9.0.0 or later is required for array support. */ static bool example(const char* host, const char* port) { diff --git a/examples/line_sender_c_example_array_elem_strides.c b/examples/line_sender_c_example_array_elem_strides.c index fcbc041d..f2a4272e 100644 --- a/examples/line_sender_c_example_array_elem_strides.c +++ b/examples/line_sender_c_example_array_elem_strides.c @@ -5,7 +5,7 @@ #include "concat.h" /* - * QuestDB server version 8.4.0 or later is required for array support. + * QuestDB server version 9.0.0 or later is required for array support. */ static bool example(const char* host, const char* port) { diff --git a/examples/line_sender_cpp_example_array_byte_strides.cpp b/examples/line_sender_cpp_example_array_byte_strides.cpp index e2ee6c62..792cead8 100644 --- a/examples/line_sender_cpp_example_array_byte_strides.cpp +++ b/examples/line_sender_cpp_example_array_byte_strides.cpp @@ -6,7 +6,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; /* - * QuestDB server version 8.4.0 or later is required for array support. + * QuestDB server version 9.0.0 or later is required for array support. */ static bool array_example(std::string_view host, std::string_view port) { diff --git a/examples/line_sender_cpp_example_array_c_major.cpp b/examples/line_sender_cpp_example_array_c_major.cpp index baced0ae..01bb869a 100644 --- a/examples/line_sender_cpp_example_array_c_major.cpp +++ b/examples/line_sender_cpp_example_array_c_major.cpp @@ -6,7 +6,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; /* - * QuestDB server version 8.4.0 or later is required for array support. + * QuestDB server version 9.0.0 or later is required for array support. */ static bool array_example(std::string_view host, std::string_view port) { diff --git a/examples/line_sender_cpp_example_array_elem_strides.cpp b/examples/line_sender_cpp_example_array_elem_strides.cpp index 8418043e..7598de48 100644 --- a/examples/line_sender_cpp_example_array_elem_strides.cpp +++ b/examples/line_sender_cpp_example_array_elem_strides.cpp @@ -6,7 +6,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; /* - * QuestDB server version 8.4.0 or later is required for array support. + * QuestDB server version 9.0.0 or later is required for array support. */ static bool array_example(std::string_view host, std::string_view port) { diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index c134d22e..b250d31c 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -115,7 +115,7 @@ typedef enum line_sender_protocol_version * Uses a binary format serialization for f64, and supports * the array data type. * This version is specific to QuestDB and not compatible with InfluxDB. - * QuestDB server version 8.4.0 or later is required for + * QuestDB server version 9.0.0 or later is required for * `line_sender_protocol_version_2` support. */ line_sender_protocol_version_2 = 2, @@ -499,7 +499,7 @@ bool line_sender_buffer_column_str( /** * Record a multidimensional array of `double` values in C-major order. * - * QuestDB server version 8.4.0 or later is required for array support. + * QuestDB server version 9.0.0 or later is required for array support. * * @param[in] buffer Line buffer object. * @param[in] name Column name. @@ -527,7 +527,7 @@ bool line_sender_buffer_column_f64_arr_c_major( * The values in the `strides` parameter represent the number of bytes * between consecutive elements along each dimension. * - * QuestDB server version 8.4.0 or later is required for array support. + * QuestDB server version 9.0.0 or later is required for array support. * * @param[in] buffer Line buffer object. * @param[in] name Column name. @@ -559,7 +559,7 @@ bool line_sender_buffer_column_f64_arr_byte_strides( * The values in the `strides` parameter represent the number of elements * between consecutive elements along each dimension. * - * QuestDB server version 8.4.0 or later is required for array support. + * QuestDB server version 9.0.0 or later is required for array support. * * @param[in] buffer Line buffer object. * @param[in] name Column name. @@ -840,7 +840,7 @@ bool line_sender_opts_token_y( * `line_sender_protocol_version_1` by default. You must explicitly set * `line_sender_protocol_version_2` in order to ingest arrays. * - * QuestDB server version 8.4.0 or later is required for + * QuestDB server version 9.0.0 or later is required for * `line_sender_protocol_version_2` support. */ LINESENDER_API diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index a9b95303..fa1ae090 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -107,7 +107,7 @@ enum class protocol_version /** * InfluxDB Line Protocol v2. - * QuestDB server version 8.4.0 or later is required for + * QuestDB server version 9.0.0 or later is required for * `v2` support. */ v2 = 2, @@ -892,7 +892,7 @@ class line_sender_buffer /** * Record a multidimensional array of `double` values. * - * QuestDB server version 8.4.0 or later is required for array support. + * QuestDB server version 9.0.0 or later is required for array support. * * @tparam T Element type (current only `double` is supported). * @tparam M Array stride size mode (bytes or elements). @@ -940,7 +940,7 @@ class line_sender_buffer * Records a multidimensional array of double-precision values with c_major * layout. * - * QuestDB server version 8.4.0 or later is required for array support. + * QuestDB server version 9.0.0 or later is required for array support. * * @tparam T Element type (current only `double` is supported). * @tparam N Number of elements in the flat data array @@ -970,7 +970,7 @@ class line_sender_buffer /** * Record a multidimensional array of double-precision values. * - * QuestDB server version 8.4.0 or later is required for array support. + * QuestDB server version 9.0.0 or later is required for array support. * * Use this method to record arrays of common or custom types such as * `std::vector`, `std::span`, `std::array`, or custom types that can be @@ -1484,7 +1484,7 @@ class opts * `protocol_version::v1` by default. You must explicitly set * `protocol_version::v2` in order to ingest arrays. * - * QuestDB server version 8.4.0 or later is required for + * QuestDB server version 9.0.0 or later is required for * `protocol_version::v2` support. */ opts& protocol_version(protocol_version version) noexcept diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index 5b54946a..03be44a7 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -307,7 +307,7 @@ pub enum ProtocolVersion { /// Version 2 of InfluxDB Line Protocol. /// Uses binary format serialization for f64, and supports the array data type. /// This version is specific to QuestDB and is not compatible with InfluxDB. - /// QuestDB server version 8.4.0 or later is required for `V2` supported. + /// QuestDB server version 9.0.0 or later is required for `V2` supported. V2 = 2, } @@ -947,7 +947,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( /// - All pointer parameters must be valid and non-null /// - shape must point to an array of `rank` integers /// - data must point to a buffer of size `data_len` f64 elements. -/// - QuestDB server version 8.4.0 or later is required for array support. +/// - QuestDB server version 9.0.0 or later is required for array support. #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_c_major( buffer: *mut line_sender_buffer, @@ -991,7 +991,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_c_major( /// - All pointer parameters must be valid and non-null /// - shape must point to an array of `rank` integers /// - data must point to a buffer of size `data_len` f64 elements. -/// - QuestDB server version 8.4.0 or later is required for array support. +/// - QuestDB server version 9.0.0 or later is required for array support. #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_byte_strides( buffer: *mut line_sender_buffer, @@ -1037,7 +1037,7 @@ pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_byte_strides( /// - All pointer parameters must be valid and non-null /// - shape must point to an array of `rank` integers /// - data must point to a buffer of size `data_len` f64 elements. -/// - QuestDB server version 8.4.0 or later is required for array support. +/// - QuestDB server version 9.0.0 or later is required for array support. #[no_mangle] pub unsafe extern "C" fn line_sender_buffer_column_f64_arr_elem_strides( buffer: *mut line_sender_buffer, @@ -1350,7 +1350,7 @@ pub unsafe extern "C" fn line_sender_opts_token_y( /// default. You must explicitly set [`ProtocolVersion::V2`] in order to ingest /// arrays. /// -/// QuestDB server version 8.4.0 or later is required for [`ProtocolVersion::V2`] support +/// QuestDB server version 9.0.0 or later is required for [`ProtocolVersion::V2`] support #[no_mangle] pub unsafe extern "C" fn line_sender_opts_protocol_version( opts: *mut line_sender_opts, diff --git a/questdb-rs/README.md b/questdb-rs/README.md index b5025852..0bcba4ee 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -25,9 +25,9 @@ These protocol versions are supported over both HTTP and TCP. | Version | Description | Server Compatibility | | ------- | ------------------------------------------------------- | --------------------- | | **1** | Over HTTP it's compatible InfluxDB Line Protocol (ILP) | All QuestDB versions | -| **2** | 64-bit floats sent as binary, adds n-dimentional arrays | 8.4.0+ (2023-10-30) | +| **2** | 64-bit floats sent as binary, adds n-dimentional arrays | 9.0.0+ (2023-10-30) | -**Note**: QuestDB server version 8.4.0 or later is required for `protocol_version=2` support. +**Note**: QuestDB server version 9.0.0 or later is required for `protocol_version=2` support. ## Quick Start @@ -58,7 +58,7 @@ fn main() -> Result<()> { .column_f64("price", 2615.54)? .column_f64("amount", 0.00044)? - // Array ingestion (QuestDB 8.4.0+). Slices and ndarray supported through trait + // Array ingestion (QuestDB 9.0.0+). Slices and ndarray supported through trait .column_arr("price_history", &[2615.54f64, 2615.10, 2614.80])? .column_arr("volatility", &ndarray::arr1(&[0.012f64, 0.011, 0.013]).view())? .at(TimestampNanos::now())?; diff --git a/questdb-rs/examples/basic.rs b/questdb-rs/examples/basic.rs index 3eca12b7..05c1f7c9 100644 --- a/questdb-rs/examples/basic.rs +++ b/questdb-rs/examples/basic.rs @@ -18,7 +18,7 @@ fn main() -> Result<()> { .symbol("side", "sell")? .column_f64("price", 2615.54)? .column_f64("amount", 0.00044)? - // QuestDB server version 8.4.0 or later is required for array support. + // QuestDB server version 9.0.0 or later is required for array support. .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? .at(designated_timestamp)?; diff --git a/questdb-rs/examples/http.rs b/questdb-rs/examples/http.rs index a59a2b53..a3f4e8d7 100644 --- a/questdb-rs/examples/http.rs +++ b/questdb-rs/examples/http.rs @@ -13,7 +13,7 @@ fn main() -> Result<()> { .symbol("side", "sell")? .column_f64("price", 2615.54)? .column_f64("amount", 0.00044)? - // QuestDB server version 8.4.0 or later is required for array support. + // QuestDB server version 9.0.0 or later is required for array support. .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? .at(TimestampNanos::now())?; sender.flush(&mut buffer)?; diff --git a/questdb-rs/examples/protocol_version.rs b/questdb-rs/examples/protocol_version.rs index 84b6ee8b..e78fe6a6 100644 --- a/questdb-rs/examples/protocol_version.rs +++ b/questdb-rs/examples/protocol_version.rs @@ -18,7 +18,7 @@ fn main() -> Result<()> { .at(TimestampNanos::now())?; sender.flush(&mut buffer)?; - // QuestDB server version 8.4.0 or later is required for `protocol_version=2` support. + // QuestDB server version 9.0.0 or later is required for `protocol_version=2` support. let mut sender2 = Sender::from_conf( "https::addr=localhost:9000;username=foo;password=bar;protocol_version=2;", )?; diff --git a/questdb-rs/src/ingress/mod.md b/questdb-rs/src/ingress/mod.md index 2ba316b0..97fa5ee6 100644 --- a/questdb-rs/src/ingress/mod.md +++ b/questdb-rs/src/ingress/mod.md @@ -265,10 +265,10 @@ arrays using several convenient types: You must use protocol version 2 to ingest arrays. HTTP transport will automatically enable it as long as you're connecting to an up-to-date QuestDB -server (version 8.4.0 or later), but with TCP you must explicitly specify it in +server (version 9.0.0 or later), but with TCP you must explicitly specify it in the configuration string: `protocol_version=2;`. -**Note**: QuestDB server version 8.4.0 or later is required for array support. +**Note**: QuestDB server version 9.0.0 or later is required for array support. ## Timestamp Column Name diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index 3fd124d9..e618ffa7 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -76,7 +76,7 @@ pub enum ProtocolVersion { /// Version 2 of InfluxDB Line Protocol. /// Uses binary format serialization for f64, and supports the array data type. /// This version is specific to QuestDB and is not compatible with InfluxDB. - /// QuestDB server version 8.4.0 or later is required for `V2` supported. + /// QuestDB server version 9.0.0 or later is required for `V2` supported. V2 = 2, } @@ -1095,7 +1095,7 @@ impl Buffer { /// Supports arrays with up to [`MAX_ARRAY_DIMS`] dimensions. The array elements must /// be of type `f64`, which is currently the only supported data type. /// - /// **Note**: QuestDB server version 8.4.0 or later is required for array support. + /// **Note**: QuestDB server version 9.0.0 or later is required for array support. /// /// # Examples /// @@ -2172,7 +2172,7 @@ impl SenderBuilder { /// default. You must explicitly set [`ProtocolVersion::V2`] in order to ingest /// arrays. /// - /// **Note**: QuestDB server version 8.4.0 or later is required for [`ProtocolVersion::V2`] support. + /// **Note**: QuestDB server version 9.0.0 or later is required for [`ProtocolVersion::V2`] support. pub fn protocol_version(mut self, protocol_version: ProtocolVersion) -> Result { self.protocol_version .set_specified("protocol_version", Some(protocol_version))?; From 75602c4256e7659a790d7378e7432516326646ee Mon Sep 17 00:00:00 2001 From: Adam Cimarosti Date: Fri, 4 Jul 2025 17:06:25 +0100 Subject: [PATCH 46/46] Refactoring and general improvements (#113) --- ci/run_all_tests.py | 21 +- ci/run_tests_pipeline.yaml | 4 +- questdb-rs-ffi/Cargo.lock | 37 +- questdb-rs-ffi/Cargo.toml | 2 +- questdb-rs/Cargo.toml | 56 +- questdb-rs/README.md | 13 +- questdb-rs/build.rs | 13 +- questdb-rs/src/error.rs | 10 +- questdb-rs/src/ingress/buffer.rs | 1275 +++++++++++ questdb-rs/src/ingress/conf.rs | 81 +- questdb-rs/src/ingress/mod.rs | 2254 +++---------------- questdb-rs/src/ingress/{ => sender}/http.rs | 84 +- questdb-rs/src/ingress/sender/mod.rs | 314 +++ questdb-rs/src/ingress/sender/tcp.rs | 245 ++ questdb-rs/src/ingress/tests.rs | 90 +- questdb-rs/src/ingress/tls.rs | 247 ++ questdb-rs/src/lib.rs | 3 + questdb-rs/src/tests/mock.rs | 67 +- questdb-rs/src/tests/mod.rs | 2 +- questdb-rs/src/tests/sender.rs | 66 +- system_test/fixture.py | 285 ++- system_test/test.py | 42 +- system_test/tls_proxy/Cargo.lock | 589 ++++- system_test/tls_proxy/Cargo.toml | 14 +- system_test/tls_proxy/src/lib.rs | 21 +- system_test/tls_proxy/src/main.rs | 2 +- 26 files changed, 3402 insertions(+), 2435 deletions(-) create mode 100644 questdb-rs/src/ingress/buffer.rs rename questdb-rs/src/ingress/{ => sender}/http.rs (89%) create mode 100644 questdb-rs/src/ingress/sender/mod.rs create mode 100644 questdb-rs/src/ingress/sender/tcp.rs create mode 100644 questdb-rs/src/ingress/tls.rs diff --git a/ci/run_all_tests.py b/ci/run_all_tests.py index f051f177..f6b47b4e 100644 --- a/ci/run_all_tests.py +++ b/ci/run_all_tests.py @@ -20,15 +20,17 @@ import pathlib import platform import subprocess +import shlex def run_cmd(*args, cwd=None): - sys.stderr.write(f'About to run: {args!r}:\n') + args_str = shlex.join(args) + sys.stderr.write(f'About to run: {args_str}:\n') try: subprocess.check_call(args, cwd=cwd) - sys.stderr.write(f'Success running: {args!r}.\n') + sys.stderr.write(f'Success running: {args_str}.\n') except subprocess.CalledProcessError as cpe: - sys.stderr.write(f'Command {args!r} failed with return code {cpe.returncode}.\n') + sys.stderr.write(f'Command `{args_str}` failed with return code {cpe.returncode}.\n') sys.exit(cpe.returncode) def main(): @@ -45,9 +47,18 @@ def main(): run_cmd('cargo', 'test', '--', '--nocapture', cwd='questdb-rs') - run_cmd('cargo', 'test', '--no-default-features', '--features=aws-lc-crypto,tls-native-certs', + run_cmd('cargo', 'test', + '--no-default-features', + '--features=aws-lc-crypto,tls-native-certs,sync-sender', + '--', '--nocapture', cwd='questdb-rs') + run_cmd('cargo', 'test', '--no-default-features', + '--features=ring-crypto,tls-native-certs,sync-sender', + '--', '--nocapture', cwd='questdb-rs') + run_cmd('cargo', 'test', '--no-default-features', + '--features=ring-crypto,tls-webpki-certs,sync-sender-tcp', '--', '--nocapture', cwd='questdb-rs') - run_cmd('cargo', 'test', '--no-default-features', '--features=ring-crypto,tls-native-certs,ilp-over-http', + run_cmd('cargo', 'test', '--no-default-features', + '--features=ring-crypto,tls-webpki-certs,sync-sender-http', '--', '--nocapture', cwd='questdb-rs') run_cmd('cargo', 'test', '--features=almost-all-features', '--', '--nocapture', cwd='questdb-rs') diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 1dd3924f..09508219 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -62,7 +62,7 @@ stages: displayName: "Compile QuestDB" inputs: mavenPOMFile: "questdb/pom.xml" - jdkVersionOption: "1.11" + jdkVersionOption: "1.17" options: "-DskipTests -Pbuild-web-console" ############################# temp for test end ##################### - script: python3 ci/run_all_tests.py @@ -132,7 +132,7 @@ stages: displayName: "Compile QuestDB" inputs: mavenPOMFile: 'questdb/pom.xml' - jdkVersionOption: '1.11' + jdkVersionOption: '1.17' options: "-DskipTests -Pbuild-web-console" - script: | python3 system_test/test.py run --repo ./questdb -v diff --git a/questdb-rs-ffi/Cargo.lock b/questdb-rs-ffi/Cargo.lock index 2d7d8799..a3773d0d 100644 --- a/questdb-rs-ffi/Cargo.lock +++ b/questdb-rs-ffi/Cargo.lock @@ -88,9 +88,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "libc", @@ -242,19 +242,18 @@ dependencies = [ [[package]] name = "r-efi" -version = "5.2.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" dependencies = [ "rand_chacha", "rand_core", - "zerocopy", ] [[package]] @@ -273,7 +272,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.2", + "getrandom 0.3.3", ] [[package]] @@ -475,26 +474,24 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "3.0.10" +version = "3.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b0351ca625c7b41a8e4f9bb6c5d9755f67f62c2187ebedecacd9974674b271d" +checksum = "9f0fde9bc91026e381155f8c67cb354bcd35260b2f4a29bcc84639f762760c39" dependencies = [ "base64", "log", "percent-encoding", - "rustls", "rustls-pemfile", "rustls-pki-types", "ureq-proto", "utf-8", - "webpki-roots", ] [[package]] name = "ureq-proto" -version = "0.3.5" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae239d0a3341aebc94259414d1dc67cfce87d41cbebc816772c91b77902fafa4" +checksum = "59db78ad1923f2b1be62b6da81fe80b173605ca0d57f85da2e005382adf693f7" dependencies = [ "base64", "http", @@ -525,9 +522,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.8" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" +checksum = "8782dd5a41a24eed3a4f40b606249b3e236ca61adf1f25ea4d45c73de122b502" dependencies = [ "rustls-pki-types", ] @@ -713,18 +710,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.24" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.24" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", diff --git a/questdb-rs-ffi/Cargo.toml b/questdb-rs-ffi/Cargo.toml index 353f6dc6..14834510 100644 --- a/questdb-rs-ffi/Cargo.toml +++ b/questdb-rs-ffi/Cargo.toml @@ -20,7 +20,7 @@ features = [ "insecure-skip-verify", "tls-webpki-certs", "tls-native-certs", - "ilp-over-http" + "sync-sender" ] [features] diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index 94244dec..84455a67 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -19,7 +19,7 @@ crate-type = ["lib"] [dependencies] libc = "0.2" -socket2 = "0.5.5" +socket2 = { version = "0.5.5", optional = true } dns-lookup = "2.0.4" base64ct = { version = "1.7", features = ["alloc"] } rustls-pemfile = "2.0.0" @@ -30,12 +30,12 @@ ring = { version = "0.17.14", optional = true } rustls-pki-types = "1.0.1" rustls = { version = "0.23.25", default-features = false, features = ["logging", "std", "tls12"] } rustls-native-certs = { version = "0.8.1", optional = true } -webpki-roots = { version = "0.26.8", default-features = false, optional = true } +webpki-roots = { version = "1.0.1", default-features = false, optional = true } chrono = { version = "0.4.40", optional = true } # We need to limit the `ureq` version to 3.0.x since we use # the `ureq::unversioned` module which does not respect semantic versioning. -ureq = { version = "3.0.10, <3.1.0", default-features = false, features = ["rustls-no-provider"], optional = true } +ureq = { version = "3.0.10, <3.1.0", default-features = false, features = ["_tls"], optional = true } serde_json = { version = "1", optional = true } questdb-confstr = "0.1.1" rand = { version = "0.9.0", optional = true } @@ -51,46 +51,60 @@ slugify = "0.1.0" indoc = "2" [dev-dependencies] +socket2 = "0.5.5" mio = { version = "1", features = ["os-poll", "net"] } chrono = "0.4.31" tempfile = "3" -webpki-roots = "0.26.8" +webpki-roots = "1.0.1" rstest = "0.25.0" [features] -default = ["tls-webpki-certs", "ilp-over-http", "ring-crypto"] +default = ["sync-sender", "tls-webpki-certs", "ring-crypto"] + +## Sync ILP/TCP + ILP/HTTP Sender +sync-sender = ["sync-sender-tcp", "sync-sender-http"] -# Include support for ILP over HTTP. -ilp-over-http = ["dep:ureq", "dep:serde_json", "dep:rand"] +## Sync ILP/TCP +sync-sender-tcp = ["_sync-sender", "_sender-tcp", "dep:socket2"] -# Allow use OS-provided root TLS certificates +## Sync ILP/HTTP +sync-sender-http = ["_sync-sender", "_sender-http", "dep:ureq", "dep:serde_json", "dep:rand"] + +## Allow use OS-provided root TLS certificates tls-native-certs = ["dep:rustls-native-certs"] -# Allow use of the `webpki-roots` crate to validate TLS certificates. +## Allow use of the `webpki-roots` crate to validate TLS certificates. tls-webpki-certs = ["dep:webpki-roots"] -# Use `aws-lc-rs` as the cryto library. +## Use `aws-lc-rs` as the cryto library. aws-lc-crypto = ["dep:aws-lc-rs", "rustls/aws-lc-rs"] -# Use `ring` as the crypto library. +## Use `ring` as the crypto library. ring-crypto = ["dep:ring", "rustls/ring"] -# Allow skipping verification of insecure certificates. +## Allow skipping verification of insecure certificates. insecure-skip-verify = [] -# Enable code-generation in `build.rs` for additional tests. +## Enable code-generation in `build.rs` for additional tests. json_tests = [] -# Enable methods to create timestamp objects from chrono::DateTime objects. +## Enable methods to create timestamp objects from chrono::DateTime objects. chrono_timestamp = ["chrono"] -# The `aws-lc-crypto` and `ring-crypto` features are mutually exclusive, -# thus compiling with `--all-features` will not work. -# Instead compile with `--features almost-all-features`. -# This is useful for quickly running `cargo test` or `cargo clippy`. +# Hidden derived features, used in code to enable-disable code sections. Don't use directly. +_sender-tcp = [] +_sender-http = [] +_sync-sender = [] + +## Enable all cross-compatible features. +## The `aws-lc-crypto` and `ring-crypto` features are mutually exclusive, +## thus compiling with `--all-features` will not work. +## Instead use `--features almost-all-features`. +## This is useful for quickly running `cargo test` or `cargo clippy`. almost-all-features = [ + "sync-sender", "tls-webpki-certs", - "ilp-over-http", + "tls-native-certs", "ring-crypto", "insecure-skip-verify", "json_tests", @@ -112,8 +126,8 @@ required-features = ["chrono_timestamp"] [[example]] name = "http" -required-features = ["ilp-over-http", "ndarray"] +required-features = ["sync-sender-http", "ndarray"] [[example]] name = "protocol_version" -required-features = ["ilp-over-http", "ndarray"] +required-features = ["sync-sender-http", "ndarray"] diff --git a/questdb-rs/README.md b/questdb-rs/README.md index 0bcba4ee..78767987 100644 --- a/questdb-rs/README.md +++ b/questdb-rs/README.md @@ -91,18 +91,19 @@ A selection of usage examples is available in the [examples directory](https://g The crate provides several optional features to enable additional functionality. You can enable features using Cargo's `--features` flag or in your `Cargo.toml`. ### Default features - -- **ilp-over-http**: Enables ILP/HTTP support via the `ureq` crate for sending data over HTTP. -- **tls-webpki-certs**: Uses the `webpki-roots` crate to validate TLS certificates. +- **sync-sender**: Enables both `sync-sender-tcp` and `sync-sender-http`. +- **sync-sender-tcp**: Enables ILP/TCP (legacy). Depends on the `socket2` crate. +- **sync-sender-http**: Enables ILP/HTTP support. Depends on the `ureq` crate. +- **tls-webpki-certs**: Uses a snapshot of the [Common CA Database](https://www.ccadb.org/) as root TLS certificates. Depends on the `webpki-roots` crate. - **ring-crypto**: Uses the `ring` crate as the cryptography backend for TLS (default crypto backend). ### Optional features -- **chrono_timestamp**: Allows specifying timestamps as `chrono::DateTime` objects. Requires the `chrono` crate. -- **tls-native-certs**: Uses OS-provided root TLS certificates for secure connections (via `rustls-native-certs`). +- **chrono_timestamp**: Allows specifying timestamps as `chrono::DateTime` objects. Depends on the `chrono` crate. +- **tls-native-certs**: Uses OS-provided root TLS certificates for secure connections. Depends on the `rustls-native-certs` crate. - **insecure-skip-verify**: Allows skipping verification of insecure certificates (not recommended for production). - **ndarray**: Enables integration with the `ndarray` crate for working with n-dimensional arrays. Without this feature, you can still send slices or implement custom array types via the `NdArrayView` trait. -- **aws-lc-crypto**: Uses `aws-lc-rs` as the cryptography backend for TLS. Mutually exclusive with `ring-crypto`. +- **aws-lc-crypto**: Uses `aws-lc-rs` as the cryptography backend for TLS. Mutually exclusive with the `ring-crypto` feature. - **almost-all-features**: Convenience feature for development and testing. Enables most features except mutually exclusive crypto backends. diff --git a/questdb-rs/build.rs b/questdb-rs/build.rs index 5ad8f336..df2d3000 100644 --- a/questdb-rs/build.rs +++ b/questdb-rs/build.rs @@ -133,9 +133,7 @@ pub mod json_tests { return true; } } - eprintln!( - "Could not match:\n {:?}\nTo any of: {:#?}", - line, expected); + eprintln!("Could not match:\n {line:?}\nTo any of: {expected:#?}"); false } "#} @@ -163,7 +161,7 @@ pub mod json_tests { writeln!(output, " || -> Result<()> {{")?; } writeln!(output, "{indent} buffer")?; - writeln!(output, "{} .table({:?})?", indent, spec.table)?; + writeln!(output, "{indent} .table({:?})?", spec.table)?; for symbol in spec.symbols.iter() { writeln!( output, @@ -216,7 +214,7 @@ pub mod json_tests { " assert_eq!(buffer.as_bytes(), exp.as_bytes());" )?; } else { - // 处理 V1 版本的 any_lines + // Checking V1 any_lines let any: Vec = expected .any_lines .as_ref() @@ -274,6 +272,11 @@ fn main() -> Result<(), Box> { "At least one of `tls-webpki-certs` or `tls-native-certs` features must be enabled." ); + #[cfg(not(any(feature = "_sender-tcp", feature = "_sender-http")))] + compile_error!( + "At least one of `sync-sender-tcp` or `sync-sender-http` features must be enabled" + ); + #[cfg(not(any(feature = "aws-lc-crypto", feature = "ring-crypto")))] compile_error!("You must enable exactly one of the `aws-lc-crypto` or `ring-crypto` features, but none are enabled."); diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index b77d6334..a28344b3 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -21,7 +21,7 @@ * limitations under the License. * ******************************************************************************/ - +use std::convert::Infallible; use std::fmt::{Display, Formatter}; macro_rules! fmt { @@ -96,7 +96,7 @@ impl Error { } } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] pub(crate) fn from_ureq_error(err: ureq::Error, url: &str) -> Error { match err { ureq::Error::StatusCode(code) => { @@ -140,6 +140,12 @@ impl Display for Error { impl std::error::Error for Error {} +impl From for Error { + fn from(_: Infallible) -> Self { + unreachable!() + } +} + /// A specialized `Result` type for the crate's [`Error`] type. pub type Result = std::result::Result; diff --git a/questdb-rs/src/ingress/buffer.rs b/questdb-rs/src/ingress/buffer.rs new file mode 100644 index 00000000..bc5ce77d --- /dev/null +++ b/questdb-rs/src/ingress/buffer.rs @@ -0,0 +1,1275 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ +use crate::ingress::ndarr::{check_and_get_array_bytes_size, ArrayElementSealed}; +use crate::ingress::{ + ndarr, ArrayElement, DebugBytes, NdArrayView, ProtocolVersion, Timestamp, TimestampMicros, + TimestampNanos, ARRAY_BINARY_FORMAT_TYPE, DOUBLE_BINARY_FORMAT_TYPE, MAX_ARRAY_DIMS, + MAX_NAME_LEN_DEFAULT, +}; +use crate::{error, Error}; +use std::fmt::{Debug, Formatter}; +use std::num::NonZeroUsize; +use std::slice::from_raw_parts_mut; + +fn write_escaped_impl(check_escape_fn: C, quoting_fn: Q, output: &mut Vec, s: &str) +where + C: Fn(u8) -> bool, + Q: Fn(&mut Vec), +{ + let mut to_escape = 0usize; + for b in s.bytes() { + if check_escape_fn(b) { + to_escape += 1; + } + } + + quoting_fn(output); + + if to_escape == 0 { + // output.push_str(s); + output.extend_from_slice(s.as_bytes()); + } else { + let additional = s.len() + to_escape; + output.reserve(additional); + let mut index = output.len(); + unsafe { output.set_len(index + additional) }; + for b in s.bytes() { + if check_escape_fn(b) { + unsafe { + *output.get_unchecked_mut(index) = b'\\'; + } + index += 1; + } + + unsafe { + *output.get_unchecked_mut(index) = b; + } + index += 1; + } + } + + quoting_fn(output); +} + +fn must_escape_unquoted(c: u8) -> bool { + matches!(c, b' ' | b',' | b'=' | b'\n' | b'\r' | b'\\') +} + +fn must_escape_quoted(c: u8) -> bool { + matches!(c, b'\n' | b'\r' | b'"' | b'\\') +} + +fn write_escaped_unquoted(output: &mut Vec, s: &str) { + write_escaped_impl(must_escape_unquoted, |_output| (), output, s); +} + +fn write_escaped_quoted(output: &mut Vec, s: &str) { + write_escaped_impl(must_escape_quoted, |output| output.push(b'"'), output, s) +} + +pub(crate) struct F64Serializer { + buf: ryu::Buffer, + n: f64, +} + +impl F64Serializer { + pub(crate) fn new(n: f64) -> Self { + F64Serializer { + buf: ryu::Buffer::new(), + n, + } + } + + // This function was taken and customized from the ryu crate. + #[cold] + fn format_nonfinite(&self) -> &'static str { + const MANTISSA_MASK: u64 = 0x000fffffffffffff; + const SIGN_MASK: u64 = 0x8000000000000000; + let bits = self.n.to_bits(); + if bits & MANTISSA_MASK != 0 { + "NaN" + } else if bits & SIGN_MASK != 0 { + "-Infinity" + } else { + "Infinity" + } + } + + pub(crate) fn as_str(&mut self) -> &str { + if self.n.is_finite() { + self.buf.format_finite(self.n) + } else { + self.format_nonfinite() + } + } +} + +#[derive(Debug, Copy, Clone)] +enum Op { + Table = 1, + Symbol = 1 << 1, + Column = 1 << 2, + At = 1 << 3, + Flush = 1 << 4, +} + +impl Op { + fn descr(self) -> &'static str { + match self { + Op::Table => "table", + Op::Symbol => "symbol", + Op::Column => "column", + Op::At => "at", + Op::Flush => "flush", + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq)] +enum OpCase { + Init = Op::Table as isize, + TableWritten = Op::Symbol as isize | Op::Column as isize, + SymbolWritten = Op::Symbol as isize | Op::Column as isize | Op::At as isize, + ColumnWritten = Op::Column as isize | Op::At as isize, + MayFlushOrTable = Op::Flush as isize | Op::Table as isize, +} + +impl OpCase { + fn next_op_descr(self) -> &'static str { + match self { + OpCase::Init => "should have called `table` instead", + OpCase::TableWritten => "should have called `symbol` or `column` instead", + OpCase::SymbolWritten => "should have called `symbol`, `column` or `at` instead", + OpCase::ColumnWritten => "should have called `column` or `at` instead", + OpCase::MayFlushOrTable => "should have called `flush` or `table` instead", + } + } +} + +// IMPORTANT: This struct MUST remain `Copy` to ensure that +// there are no heap allocations when performing marker operations. +#[derive(Debug, Clone, Copy)] +struct BufferState { + op_case: OpCase, + row_count: usize, + first_table_len: Option, + transactional: bool, +} + +impl BufferState { + fn new() -> Self { + Self { + op_case: OpCase::Init, + row_count: 0, + first_table_len: None, + transactional: true, + } + } +} + +/// A validated table name. +/// +/// This type simply wraps a `&str`. +/// +/// When you pass a `TableName` instead of a plain string to a [`Buffer`] method, +/// it doesn't have to validate it again. This saves CPU cycles. +#[derive(Clone, Copy)] +pub struct TableName<'a> { + name: &'a str, +} + +impl<'a> TableName<'a> { + /// Construct a validated table name. + pub fn new(name: &'a str) -> crate::Result { + if name.is_empty() { + return Err(error::fmt!( + InvalidName, + "Table names must have a non-zero length." + )); + } + + let mut prev = '\0'; + for (index, c) in name.chars().enumerate() { + match c { + '.' => { + if index == 0 || index == name.len() - 1 || prev == '.' { + return Err(error::fmt!( + InvalidName, + concat!("Bad string {:?}: ", "Found invalid dot `.` at position {}."), + name, + index + )); + } + } + '?' | ',' | '\'' | '\"' | '\\' | '/' | ':' | ')' | '(' | '+' | '*' | '%' | '~' + | '\r' | '\n' | '\0' | '\u{0001}' | '\u{0002}' | '\u{0003}' | '\u{0004}' + | '\u{0005}' | '\u{0006}' | '\u{0007}' | '\u{0008}' | '\u{0009}' | '\u{000b}' + | '\u{000c}' | '\u{000e}' | '\u{000f}' | '\u{007f}' => { + return Err(error::fmt!( + InvalidName, + concat!( + "Bad string {:?}: ", + "Table names can't contain ", + "a {:?} character, which was found at ", + "byte position {}." + ), + name, + c, + index + )); + } + '\u{feff}' => { + // Reject Unicode char 'ZERO WIDTH NO-BREAK SPACE', + // aka UTF-8 BOM if it appears anywhere in the string. + return Err(error::fmt!( + InvalidName, + concat!( + "Bad string {:?}: ", + "Table names can't contain ", + "a UTF-8 BOM character, which was found at ", + "byte position {}." + ), + name, + index + )); + } + _ => (), + } + prev = c; + } + + Ok(Self { name }) + } + + /// Construct a table name without validating it. + /// + /// This breaks API encapsulation and is only intended for use + /// when the string was already previously validated. + /// + /// The QuestDB server will reject an invalid table name. + pub fn new_unchecked(name: &'a str) -> Self { + Self { name } + } +} + +impl<'a> TryFrom<&'a str> for TableName<'a> { + type Error = Error; + + fn try_from(name: &'a str) -> crate::Result { + Self::new(name) + } +} + +impl<'a> AsRef for TableName<'a> { + fn as_ref(&self) -> &str { + self.name + } +} + +/// A validated column name. +/// +/// This type simply wraps a `&str`. +/// +/// When you pass a `ColumnName` instead of a plain string to a [`Buffer`] method, +/// it doesn't have to validate it again. This saves CPU cycles. +#[derive(Clone, Copy)] +pub struct ColumnName<'a> { + name: &'a str, +} + +impl<'a> ColumnName<'a> { + /// Construct a validated table name. + pub fn new(name: &'a str) -> crate::Result { + if name.is_empty() { + return Err(error::fmt!( + InvalidName, + "Column names must have a non-zero length." + )); + } + + for (index, c) in name.chars().enumerate() { + match c { + '?' | '.' | ',' | '\'' | '\"' | '\\' | '/' | ':' | ')' | '(' | '+' | '-' | '*' + | '%' | '~' | '\r' | '\n' | '\0' | '\u{0001}' | '\u{0002}' | '\u{0003}' + | '\u{0004}' | '\u{0005}' | '\u{0006}' | '\u{0007}' | '\u{0008}' | '\u{0009}' + | '\u{000b}' | '\u{000c}' | '\u{000e}' | '\u{000f}' | '\u{007f}' => { + return Err(error::fmt!( + InvalidName, + concat!( + "Bad string {:?}: ", + "Column names can't contain ", + "a {:?} character, which was found at ", + "byte position {}." + ), + name, + c, + index + )); + } + '\u{FEFF}' => { + // Reject Unicode char 'ZERO WIDTH NO-BREAK SPACE', + // aka UTF-8 BOM if it appears anywhere in the string. + return Err(error::fmt!( + InvalidName, + concat!( + "Bad string {:?}: ", + "Column names can't contain ", + "a UTF-8 BOM character, which was found at ", + "byte position {}." + ), + name, + index + )); + } + _ => (), + } + } + + Ok(Self { name }) + } + + /// Construct a column name without validating it. + /// + /// This breaks API encapsulation and is only intended for use + /// when the string was already previously validated. + /// + /// The QuestDB server will reject an invalid column name. + pub fn new_unchecked(name: &'a str) -> Self { + Self { name } + } +} + +impl<'a> TryFrom<&'a str> for ColumnName<'a> { + type Error = Error; + + fn try_from(name: &'a str) -> crate::Result { + Self::new(name) + } +} + +impl<'a> AsRef for ColumnName<'a> { + fn as_ref(&self) -> &str { + self.name + } +} + +/// A reusable buffer to prepare a batch of ILP messages. +/// +/// # Example +/// +/// ```no_run +/// # use questdb::Result; +/// # use questdb::ingress::SenderBuilder; +/// +/// # fn main() -> Result<()> { +/// # let mut sender = SenderBuilder::from_conf("http::addr=localhost:9000;")?.build()?; +/// # use questdb::Result; +/// use questdb::ingress::{Buffer, TimestampMicros, TimestampNanos}; +/// let mut buffer = sender.new_buffer(); +/// +/// // first row +/// buffer +/// .table("table1")? +/// .symbol("bar", "baz")? +/// .column_bool("a", false)? +/// .column_i64("b", 42)? +/// .column_f64("c", 3.14)? +/// .column_str("d", "hello")? +/// .column_ts("e", TimestampMicros::now())? +/// .at(TimestampNanos::now())?; +/// +/// // second row +/// buffer +/// .table("table2")? +/// .symbol("foo", "bar")? +/// .at(TimestampNanos::now())?; +/// # Ok(()) +/// # } +/// ``` +/// +/// Send the buffer to QuestDB using [`sender.flush(&mut buffer)`](Sender::flush). +/// +/// # Sequential Coupling +/// The Buffer API is sequentially coupled: +/// * A row always starts with [`table`](Buffer::table). +/// * A row must contain at least one [`symbol`](Buffer::symbol) or +/// column ( +/// [`column_bool`](Buffer::column_bool), +/// [`column_i64`](Buffer::column_i64), +/// [`column_f64`](Buffer::column_f64), +/// [`column_str`](Buffer::column_str), +/// [`column_arr`](Buffer::column_arr), +/// [`column_ts`](Buffer::column_ts)). +/// * Symbols must appear before columns. +/// * A row must be terminated with either [`at`](Buffer::at) or +/// [`at_now`](Buffer::at_now). +/// +/// This diagram visualizes the sequence: +/// +/// +/// +/// # Buffer method calls, Serialized ILP types and QuestDB types +/// +/// | Buffer Method | Serialized as ILP type (Click on link to see possible casts) | +/// |---------------|--------------------------------------------------------------| +/// | [`symbol`](Buffer::symbol) | [`SYMBOL`](https://questdb.io/docs/concept/symbol/) | +/// | [`column_bool`](Buffer::column_bool) | [`BOOLEAN`](https://questdb.io/docs/reference/api/ilp/columnset-types#boolean) | +/// | [`column_i64`](Buffer::column_i64) | [`INTEGER`](https://questdb.io/docs/reference/api/ilp/columnset-types#integer) | +/// | [`column_f64`](Buffer::column_f64) | [`FLOAT`](https://questdb.io/docs/reference/api/ilp/columnset-types#float) | +/// | [`column_str`](Buffer::column_str) | [`STRING`](https://questdb.io/docs/reference/api/ilp/columnset-types#string) | +/// | [`column_arr`](Buffer::column_arr) | [`ARRAY`](https://questdb.io/docs/reference/api/ilp/columnset-types#array) | +/// | [`column_ts`](Buffer::column_ts) | [`TIMESTAMP`](https://questdb.io/docs/reference/api/ilp/columnset-types#timestamp) | +/// +/// QuestDB supports both `STRING` and `SYMBOL` column types. +/// +/// To understand the difference, refer to the +/// [QuestDB documentation](https://questdb.io/docs/concept/symbol/). In a nutshell, +/// symbols are interned strings, most suitable for identifiers that are repeated many +/// times throughout the column. They offer an advantage in storage space and query +/// performance. +/// +/// # Inserting NULL values +/// +/// To insert a NULL value, skip the symbol or column for that row. +/// +/// # Recovering from validation errors +/// +/// If you want to recover from potential validation errors, call +/// [`buffer.set_marker()`](Buffer::set_marker) to track the last known good state, +/// append as many rows or parts of rows as you like, and then call +/// [`buffer.clear_marker()`](Buffer::clear_marker) on success. +/// +/// If there was an error in one of the rows, use +/// [`buffer.rewind_to_marker()`](Buffer::rewind_to_marker) to go back to the +/// marked last known good state. +/// +#[derive(Clone)] +pub struct Buffer { + output: Vec, + state: BufferState, + marker: Option<(usize, BufferState)>, + max_name_len: usize, + protocol_version: ProtocolVersion, +} + +impl Buffer { + /// Creates a new [`Buffer`] with default parameters. + /// + /// - Uses the specified protocol version + /// - Sets maximum name length to **127 characters** (QuestDB server default) + /// + /// This is equivalent to [`Sender::new_buffer`] when using the [`Sender::protocol_version`] + /// and [`Sender::max_name_len`] is 127. + /// + /// For custom name lengths, use [`Self::with_max_name_len`] + pub fn new(protocol_version: ProtocolVersion) -> Self { + Self::with_max_name_len(protocol_version, MAX_NAME_LEN_DEFAULT) + } + + /// Creates a new [`Buffer`] with a custom maximum name length. + /// + /// - `max_name_len`: Maximum allowed length for table/column names, match + /// your QuestDB server's `cairo.max.file.name.length` configuration + /// - `protocol_version`: Protocol version to use + /// + /// This is equivalent to [`Sender::new_buffer`] when using the [`Sender::protocol_version`] + /// and [`Sender::max_name_len`]. + /// + /// For the default max name length limit (127), use [`Self::new`]. + pub fn with_max_name_len(protocol_version: ProtocolVersion, max_name_len: usize) -> Self { + Self { + output: Vec::new(), + state: BufferState::new(), + marker: None, + max_name_len, + protocol_version, + } + } + + pub fn protocol_version(&self) -> ProtocolVersion { + self.protocol_version + } + + /// Pre-allocate to ensure the buffer has enough capacity for at least the + /// specified additional byte count. This may be rounded up. + /// This does not allocate if such additional capacity is already satisfied. + /// See: `capacity`. + pub fn reserve(&mut self, additional: usize) { + self.output.reserve(additional); + } + + /// The number of bytes accumulated in the buffer. + pub fn len(&self) -> usize { + self.output.len() + } + + /// The number of rows accumulated in the buffer. + pub fn row_count(&self) -> usize { + self.state.row_count + } + + /// Tells whether the buffer is transactional. It is transactional iff it contains + /// data for at most one table. Additionally, you must send the buffer over HTTP to + /// get transactional behavior. + pub fn transactional(&self) -> bool { + self.state.transactional + } + + pub fn is_empty(&self) -> bool { + self.output.is_empty() + } + + /// The total number of bytes the buffer can hold before it needs to resize. + pub fn capacity(&self) -> usize { + self.output.capacity() + } + + pub fn as_bytes(&self) -> &[u8] { + &self.output + } + + /// Mark a rewind point. + /// This allows undoing accumulated changes to the buffer for one or more + /// rows by calling [`rewind_to_marker`](Buffer::rewind_to_marker). + /// Any previous marker will be discarded. + /// Once the marker is no longer needed, call + /// [`clear_marker`](Buffer::clear_marker). + pub fn set_marker(&mut self) -> crate::Result<()> { + if (self.state.op_case as isize & Op::Table as isize) == 0 { + return Err(error::fmt!( + InvalidApiCall, + concat!( + "Can't set the marker whilst constructing a line. ", + "A marker may only be set on an empty buffer or after ", + "`at` or `at_now` is called." + ) + )); + } + self.marker = Some((self.output.len(), self.state)); + Ok(()) + } + + /// Undo all changes since the last [`set_marker`](Buffer::set_marker) + /// call. + /// + /// As a side effect, this also clears the marker. + pub fn rewind_to_marker(&mut self) -> crate::Result<()> { + if let Some((position, state)) = self.marker.take() { + self.output.truncate(position); + self.state = state; + Ok(()) + } else { + Err(error::fmt!( + InvalidApiCall, + "Can't rewind to the marker: No marker set." + )) + } + } + + /// Discard any marker as may have been set by + /// [`set_marker`](Buffer::set_marker). + /// + /// Idempotent. + pub fn clear_marker(&mut self) { + self.marker = None; + } + + /// Reset the buffer and clear contents whilst retaining + /// [`capacity`](Buffer::capacity). + pub fn clear(&mut self) { + self.output.clear(); + self.state = BufferState::new(); + self.marker = None; + } + + /// Check if the next API operation is allowed as per the OP case state machine. + #[inline(always)] + fn check_op(&self, op: Op) -> crate::Result<()> { + if (self.state.op_case as isize & op as isize) > 0 { + Ok(()) + } else { + Err(error::fmt!( + InvalidApiCall, + "State error: Bad call to `{}`, {}.", + op.descr(), + self.state.op_case.next_op_descr() + )) + } + } + + /// Checks if this buffer is ready to be flushed to a sender via one of the + /// [`Sender::flush`] functions. An [`Ok`] value indicates that the buffer + /// is ready to be flushed via a [`Sender`] while an [`Err`] will contain a + /// message indicating why this [`Buffer`] cannot be flushed at the moment. + #[inline(always)] + pub fn check_can_flush(&self) -> crate::Result<()> { + self.check_op(Op::Flush) + } + + #[inline(always)] + fn validate_max_name_len(&self, name: &str) -> crate::Result<()> { + if name.len() > self.max_name_len { + return Err(error::fmt!( + InvalidName, + "Bad name: {:?}: Too long (max {} characters)", + name, + self.max_name_len + )); + } + Ok(()) + } + + /// Begin recording a new row for the given table. + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// buffer.table("table_name")?; + /// # Ok(()) + /// # } + /// ``` + /// + /// or + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use questdb::ingress::TableName; + /// + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// let table_name = TableName::new("table_name")?; + /// buffer.table(table_name)?; + /// # Ok(()) + /// # } + /// ``` + pub fn table<'a, N>(&mut self, name: N) -> crate::Result<&mut Self> + where + N: TryInto>, + Error: From, + { + let name: TableName<'a> = name.try_into()?; + self.validate_max_name_len(name.name)?; + self.check_op(Op::Table)?; + let table_begin = self.output.len(); + write_escaped_unquoted(&mut self.output, name.name); + let table_end = self.output.len(); + self.state.op_case = OpCase::TableWritten; + + // A buffer stops being transactional if it targets multiple tables. + if let Some(first_table_len) = &self.state.first_table_len { + let first_table = &self.output[0..first_table_len.get()]; + let this_table = &self.output[table_begin..table_end]; + if first_table != this_table { + self.state.transactional = false; + } + } else { + debug_assert!(table_begin == 0); + + // This is a bit confusing, so worth explaining: + // `NonZeroUsize::new(table_end)` will return `None` if `table_end` is 0, + // but we know that `table_end` is never 0 here, we just need an option type + // anyway, so we don't bother unwrapping it to then wrap it again. + let first_table_len = NonZeroUsize::new(table_end); + + // Instead we just assert that it's `Some`. + debug_assert!(first_table_len.is_some()); + + self.state.first_table_len = first_table_len; + } + Ok(self) + } + + /// Record a symbol for the given column. + /// Make sure you record all symbol columns before any other column type. + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// buffer.symbol("col_name", "value")?; + /// # Ok(()) + /// # } + /// ``` + /// + /// or + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let value: String = "value".to_owned(); + /// buffer.symbol("col_name", value)?; + /// # Ok(()) + /// # } + /// ``` + /// + /// or + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use questdb::ingress::ColumnName; + /// + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let col_name = ColumnName::new("col_name")?; + /// buffer.symbol(col_name, "value")?; + /// # Ok(()) + /// # } + /// ``` + /// + pub fn symbol<'a, N, S>(&mut self, name: N, value: S) -> crate::Result<&mut Self> + where + N: TryInto>, + S: AsRef, + Error: From, + { + let name: ColumnName<'a> = name.try_into()?; + self.validate_max_name_len(name.name)?; + self.check_op(Op::Symbol)?; + self.output.push(b','); + write_escaped_unquoted(&mut self.output, name.name); + self.output.push(b'='); + write_escaped_unquoted(&mut self.output, value.as_ref()); + self.state.op_case = OpCase::SymbolWritten; + Ok(self) + } + + fn write_column_key<'a, N>(&mut self, name: N) -> crate::Result<&mut Self> + where + N: TryInto>, + Error: From, + { + let name: ColumnName<'a> = name.try_into()?; + self.validate_max_name_len(name.name)?; + self.check_op(Op::Column)?; + self.output + .push(if (self.state.op_case as isize & Op::Symbol as isize) > 0 { + b' ' + } else { + b',' + }); + write_escaped_unquoted(&mut self.output, name.name); + self.output.push(b'='); + self.state.op_case = OpCase::ColumnWritten; + Ok(self) + } + + /// Record a boolean value for the given column. + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// buffer.column_bool("col_name", true)?; + /// # Ok(()) + /// # } + /// ``` + /// + /// or + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use questdb::ingress::ColumnName; + /// + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let col_name = ColumnName::new("col_name")?; + /// buffer.column_bool(col_name, true)?; + /// # Ok(()) + /// # } + /// ``` + pub fn column_bool<'a, N>(&mut self, name: N, value: bool) -> crate::Result<&mut Self> + where + N: TryInto>, + Error: From, + { + self.write_column_key(name)?; + self.output.push(if value { b't' } else { b'f' }); + Ok(self) + } + + /// Record an integer value for the given column. + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// buffer.column_i64("col_name", 42)?; + /// # Ok(()) + /// # } + /// ``` + /// + /// or + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use questdb::ingress::ColumnName; + /// + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let col_name = ColumnName::new("col_name")?; + /// buffer.column_i64(col_name, 42)?; + /// # Ok(()) + /// # } + /// ``` + pub fn column_i64<'a, N>(&mut self, name: N, value: i64) -> crate::Result<&mut Self> + where + N: TryInto>, + Error: From, + { + self.write_column_key(name)?; + let mut buf = itoa::Buffer::new(); + let printed = buf.format(value); + self.output.extend_from_slice(printed.as_bytes()); + self.output.push(b'i'); + Ok(self) + } + + /// Record a floating point value for the given column. + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// buffer.column_f64("col_name", 3.14)?; + /// # Ok(()) + /// # } + /// ``` + /// + /// or + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use questdb::ingress::ColumnName; + /// + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let col_name = ColumnName::new("col_name")?; + /// buffer.column_f64(col_name, 3.14)?; + /// # Ok(()) + /// # } + /// ``` + pub fn column_f64<'a, N>(&mut self, name: N, value: f64) -> crate::Result<&mut Self> + where + N: TryInto>, + Error: From, + { + self.write_column_key(name)?; + if !matches!(self.protocol_version, ProtocolVersion::V1) { + self.output.push(b'='); + self.output.push(DOUBLE_BINARY_FORMAT_TYPE); + self.output.extend_from_slice(&value.to_le_bytes()) + } else { + let mut ser = F64Serializer::new(value); + self.output.extend_from_slice(ser.as_str().as_bytes()) + } + Ok(self) + } + + /// Record a string value for the given column. + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// buffer.column_str("col_name", "value")?; + /// # Ok(()) + /// # } + /// ``` + /// + /// or + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let value: String = "value".to_owned(); + /// buffer.column_str("col_name", value)?; + /// # Ok(()) + /// # } + /// ``` + /// + /// or + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use questdb::ingress::ColumnName; + /// + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let col_name = ColumnName::new("col_name")?; + /// buffer.column_str(col_name, "value")?; + /// # Ok(()) + /// # } + /// ``` + pub fn column_str<'a, N, S>(&mut self, name: N, value: S) -> crate::Result<&mut Self> + where + N: TryInto>, + S: AsRef, + Error: From, + { + self.write_column_key(name)?; + write_escaped_quoted(&mut self.output, value.as_ref()); + Ok(self) + } + + /// Record a multidimensional array value for the given column. + /// + /// Supports arrays with up to [`MAX_ARRAY_DIMS`] dimensions. The array elements must + /// be of type `f64`, which is currently the only supported data type. + /// + /// **Note**: QuestDB server version 9.0.0 or later is required for array support. + /// + /// # Examples + /// + /// Recording a 2D array using slices: + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let array_2d = vec![vec![1.1, 2.2], vec![3.3, 4.4]]; + /// buffer.column_arr("array_col", &array_2d)?; + /// # Ok(()) + /// # } + /// ``` + /// + /// Recording a 3D array using vectors: + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, ColumnName, SenderBuilder}; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x1")?; + /// let array_3d = vec![vec![vec![42.0; 4]; 3]; 2]; + /// let col_name = ColumnName::new("col1")?; + /// buffer.column_arr(col_name, &array_3d)?; + /// # Ok(()) + /// # } + /// ``` + /// + /// # Errors + /// + /// Returns [`Error`] if: + /// - Array dimensions exceed [`MAX_ARRAY_DIMS`] + /// - Failed to get dimension sizes + /// - Column name validation fails + /// - Protocol version v1 is used (arrays require v2+) + #[allow(private_bounds)] + pub fn column_arr<'a, N, T, D>(&mut self, name: N, view: &T) -> crate::Result<&mut Self> + where + N: TryInto>, + T: NdArrayView, + D: ArrayElement + ArrayElementSealed, + Error: From, + { + if self.protocol_version == ProtocolVersion::V1 { + return Err(error::fmt!( + ProtocolVersionError, + "Protocol version v1 does not support array datatype", + )); + } + let ndim = view.ndim(); + if ndim == 0 { + return Err(error::fmt!( + ArrayError, + "Zero-dimensional arrays are not supported", + )); + } + + // check dimension less equal than max dims + if MAX_ARRAY_DIMS < ndim { + return Err(error::fmt!( + ArrayError, + "Array dimension mismatch: expected at most {} dimensions, but got {}", + MAX_ARRAY_DIMS, + ndim + )); + } + + let array_buf_size = check_and_get_array_bytes_size(view)?; + self.write_column_key(name)?; + // binary format flag '=' + self.output.push(b'='); + // binary format entity type + self.output.push(ARRAY_BINARY_FORMAT_TYPE); + // ndarr datatype + self.output.push(D::type_tag()); + // ndarr dims + self.output.push(ndim as u8); + + let dim_header_size = size_of::() * ndim; + self.output.reserve(dim_header_size + array_buf_size); + + for i in 0..ndim { + // ndarr shape + self.output + .extend_from_slice((view.dim(i)? as u32).to_le_bytes().as_slice()); + } + + let index = self.output.len(); + let writeable = + unsafe { from_raw_parts_mut(self.output.as_mut_ptr().add(index), array_buf_size) }; + + // ndarr data + ndarr::write_array_data(view, writeable, array_buf_size)?; + unsafe { self.output.set_len(array_buf_size + index) } + Ok(self) + } + + /// Record a timestamp value for the given column. + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use questdb::ingress::TimestampMicros; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// buffer.column_ts("col_name", TimestampMicros::now())?; + /// # Ok(()) + /// # } + /// ``` + /// + /// or + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use questdb::ingress::TimestampMicros; + /// + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// buffer.column_ts("col_name", TimestampMicros::new(1659548204354448))?; + /// # Ok(()) + /// # } + /// ``` + /// + /// or + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use questdb::ingress::TimestampMicros; + /// use questdb::ingress::ColumnName; + /// + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let col_name = ColumnName::new("col_name")?; + /// buffer.column_ts(col_name, TimestampMicros::now())?; + /// # Ok(()) + /// # } + /// ``` + /// + /// or you can also pass in a `TimestampNanos`. + /// + /// Note that both `TimestampMicros` and `TimestampNanos` can be constructed + /// easily from either `std::time::SystemTime` or `chrono::DateTime`. + /// + /// This last option requires the `chrono_timestamp` feature. + pub fn column_ts<'a, N, T>(&mut self, name: N, value: T) -> crate::Result<&mut Self> + where + N: TryInto>, + T: TryInto, + Error: From, + Error: From, + { + self.write_column_key(name)?; + let timestamp: Timestamp = value.try_into()?; + let timestamp: TimestampMicros = timestamp.try_into()?; + let mut buf = itoa::Buffer::new(); + let printed = buf.format(timestamp.as_i64()); + self.output.extend_from_slice(printed.as_bytes()); + self.output.push(b't'); + Ok(self) + } + + /// Complete the current row with the designated timestamp. After this call, you can + /// start recording the next row by calling [Buffer::table] again, or you can send + /// the accumulated batch by calling [Sender::flush] or one of its variants. + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use questdb::ingress::TimestampNanos; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?.symbol("a", "b")?; + /// buffer.at(TimestampNanos::now())?; + /// # Ok(()) + /// # } + /// ``` + /// + /// or + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use questdb::ingress::TimestampNanos; + /// + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?.symbol("a", "b")?; + /// buffer.at(TimestampNanos::new(1659548315647406592))?; + /// # Ok(()) + /// # } + /// ``` + /// + /// You can also pass in a `TimestampMicros`. + /// + /// Note that both `TimestampMicros` and `TimestampNanos` can be constructed + /// easily from either `std::time::SystemTime` or `chrono::DateTime`. + /// + pub fn at(&mut self, timestamp: T) -> crate::Result<()> + where + T: TryInto, + Error: From, + { + self.check_op(Op::At)?; + let timestamp: Timestamp = timestamp.try_into()?; + + // https://github.com/rust-lang/rust/issues/115880 + let timestamp: crate::Result = timestamp.try_into(); + let timestamp: TimestampNanos = timestamp?; + + let epoch_nanos = timestamp.as_i64(); + if epoch_nanos < 0 { + return Err(error::fmt!( + InvalidTimestamp, + "Timestamp {} is negative. It must be >= 0.", + epoch_nanos + )); + } + let mut buf = itoa::Buffer::new(); + let printed = buf.format(epoch_nanos); + self.output.push(b' '); + self.output.extend_from_slice(printed.as_bytes()); + self.output.push(b'\n'); + self.state.op_case = OpCase::MayFlushOrTable; + self.state.row_count += 1; + Ok(()) + } + + /// Complete the current row without providing a timestamp. The QuestDB instance + /// will insert its own timestamp. + /// + /// Letting the server assign the timestamp can be faster since it reliably avoids + /// out-of-order operations in the database for maximum ingestion throughput. However, + /// it removes the ability to deduplicate rows. + /// + /// This is NOT equivalent to calling [Buffer::at] with the current time: the QuestDB + /// server will set the timestamp only after receiving the row. If you're flushing + /// infrequently, the server-assigned timestamp may be significantly behind the + /// time the data was recorded in the buffer. + /// + /// In almost all cases, you should prefer the [Buffer::at] function. + /// + /// After this call, you can start recording the next row by calling [Buffer::table] + /// again, or you can send the accumulated batch by calling [Sender::flush] or one of + /// its variants. + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?.symbol("a", "b")?; + /// buffer.at_now()?; + /// # Ok(()) + /// # } + /// ``` + pub fn at_now(&mut self) -> crate::Result<()> { + self.check_op(Op::At)?; + self.output.push(b'\n'); + self.state.op_case = OpCase::MayFlushOrTable; + self.state.row_count += 1; + Ok(()) + } +} + +impl Debug for Buffer { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Buffer") + .field("output", &DebugBytes(&self.output)) + .field("state", &self.state) + .field("marker", &self.marker) + .field("max_name_len", &self.max_name_len) + .field("protocol_version", &self.protocol_version) + .finish() + } +} diff --git a/questdb-rs/src/ingress/conf.rs b/questdb-rs/src/ingress/conf.rs index f5a9e272..8e3ac4a4 100644 --- a/questdb-rs/src/ingress/conf.rs +++ b/questdb-rs/src/ingress/conf.rs @@ -22,9 +22,8 @@ * ******************************************************************************/ -use std::ops::Deref; - use crate::{Error, ErrorCode, Result}; +use std::ops::Deref; /// Wraps a SenderBuilder config setting with the intent of tracking /// whether the value was user-specified or defaulted. @@ -73,3 +72,81 @@ impl Deref for ConfigSetting { } } } + +#[cfg(feature = "_sender-http")] +#[derive(Debug, Clone)] +pub(crate) struct HttpConfig { + pub(crate) request_min_throughput: ConfigSetting, + pub(crate) user_agent: String, + pub(crate) retry_timeout: ConfigSetting, + pub(crate) request_timeout: ConfigSetting, +} + +#[cfg(feature = "_sender-http")] +impl Default for HttpConfig { + fn default() -> Self { + Self { + request_min_throughput: ConfigSetting::new_default(102400), // 100 KiB/s + user_agent: concat!("questdb/rust/", env!("CARGO_PKG_VERSION")).to_string(), + retry_timeout: ConfigSetting::new_default(std::time::Duration::from_secs(10)), + request_timeout: ConfigSetting::new_default(std::time::Duration::from_secs(10)), + } + } +} + +#[cfg(feature = "_sender-http")] +#[derive(PartialEq, Debug, Clone)] +pub(crate) struct BasicAuthParams { + pub(crate) username: String, + pub(crate) password: String, +} + +#[cfg(feature = "_sender-http")] +impl BasicAuthParams { + pub(crate) fn to_header_string(&self) -> String { + use base64ct::{Base64, Encoding}; + let pair = format!("{}:{}", self.username, self.password); + let encoded = Base64::encode_string(pair.as_bytes()); + format!("Basic {encoded}") + } +} + +#[cfg(feature = "_sender-http")] +#[derive(PartialEq, Debug, Clone)] +pub(crate) struct TokenAuthParams { + pub(crate) token: String, +} + +#[cfg(feature = "_sender-http")] +impl TokenAuthParams { + pub(crate) fn to_header_string(&self) -> crate::Result { + if self.token.contains('\n') { + return Err(crate::error::fmt!( + AuthError, + "Bad auth token: Should not contain new-line char." + )); + } + Ok(format!("Bearer {}", self.token)) + } +} + +#[cfg(feature = "_sender-tcp")] +#[derive(PartialEq, Debug, Clone)] +pub(crate) struct EcdsaAuthParams { + pub(crate) key_id: String, + pub(crate) priv_key: String, + pub(crate) pub_key_x: String, + pub(crate) pub_key_y: String, +} + +#[derive(PartialEq, Debug, Clone)] +pub(crate) enum AuthParams { + #[cfg(feature = "_sender-tcp")] + Ecdsa(EcdsaAuthParams), + + #[cfg(feature = "_sender-http")] + Basic(BasicAuthParams), + + #[cfg(feature = "_sender-http")] + Token(TokenAuthParams), +} diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index e618ffa7..473e6e13 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -26,38 +26,42 @@ pub use self::ndarr::{ArrayElement, NdArrayView}; pub use self::timestamp::*; -use crate::error::{self, Error, Result}; -use crate::gai; +use crate::error::{self, fmt, Result}; use crate::ingress::conf::ConfigSetting; -use base64ct::{Base64, Base64UrlUnpadded, Encoding}; use core::time::Duration; -use ndarr::ArrayElementSealed; -use rustls::{ClientConnection, RootCertStore, StreamOwned}; -use rustls_pki_types::ServerName; -use socket2::{Domain, Protocol as SockProtocol, SockAddr, Socket, Type}; use std::collections::HashMap; -use std::convert::Infallible; use std::fmt::{Debug, Display, Formatter, Write}; -use std::io::{self, BufRead, BufReader, ErrorKind, Write as IoWrite}; -use std::num::NonZeroUsize; + use std::ops::Deref; use std::path::PathBuf; -use std::slice::from_raw_parts_mut; use std::str::FromStr; -use std::sync::Arc; -#[cfg(feature = "aws-lc-crypto")] +mod tls; + +#[cfg(all(feature = "_sender-tcp", feature = "aws-lc-crypto"))] use aws_lc_rs::{ rand::SystemRandom, signature::{EcdsaKeyPair, ECDSA_P256_SHA256_FIXED_SIGNING}, }; -#[cfg(feature = "ring-crypto")] +#[cfg(all(feature = "_sender-tcp", feature = "ring-crypto"))] use ring::{ rand::SystemRandom, signature::{EcdsaKeyPair, ECDSA_P256_SHA256_FIXED_SIGNING}, }; +mod conf; + +pub(crate) mod ndarr; + +mod timestamp; + +mod buffer; +pub use buffer::*; + +mod sender; +pub use sender::*; + const MAX_NAME_LEN_DEFAULT: usize = 127; /// The maximum allowed dimensions for arrays. @@ -65,6 +69,9 @@ pub const MAX_ARRAY_DIMS: usize = 32; pub const MAX_ARRAY_BUFFER_SIZE: usize = 512 * 1024 * 1024; // 512MiB pub const MAX_ARRAY_DIM_LEN: usize = 0x0FFF_FFFF; // 1 << 28 - 1 +pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; +pub(crate) const DOUBLE_BINARY_FORMAT_TYPE: u8 = 16; + /// The version of InfluxDB Line Protocol used to communicate with the server. #[derive(Debug, Copy, Clone, PartialEq)] pub enum ProtocolVersion { @@ -80,1340 +87,18 @@ pub enum ProtocolVersion { V2 = 2, } -impl std::fmt::Display for ProtocolVersion { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl Display for ProtocolVersion { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { ProtocolVersion::V1 => write!(f, "v1"), ProtocolVersion::V2 => write!(f, "v2"), } - } -} - -#[derive(Debug, Copy, Clone)] -enum Op { - Table = 1, - Symbol = 1 << 1, - Column = 1 << 2, - At = 1 << 3, - Flush = 1 << 4, -} - -impl Op { - fn descr(self) -> &'static str { - match self { - Op::Table => "table", - Op::Symbol => "symbol", - Op::Column => "column", - Op::At => "at", - Op::Flush => "flush", - } - } -} - -fn map_io_to_socket_err(prefix: &str, io_err: io::Error) -> Error { - error::fmt!(SocketError, "{}{}", prefix, io_err) -} - -/// A validated table name. -/// -/// This type simply wraps a `&str`. -/// -/// When you pass a `TableName` instead of a plain string to a [`Buffer`] method, -/// it doesn't have to validate it again. This saves CPU cycles. -#[derive(Clone, Copy)] -pub struct TableName<'a> { - name: &'a str, -} - -impl<'a> TableName<'a> { - /// Construct a validated table name. - pub fn new(name: &'a str) -> Result { - if name.is_empty() { - return Err(error::fmt!( - InvalidName, - "Table names must have a non-zero length." - )); - } - - let mut prev = '\0'; - for (index, c) in name.chars().enumerate() { - match c { - '.' => { - if index == 0 || index == name.len() - 1 || prev == '.' { - return Err(error::fmt!( - InvalidName, - concat!("Bad string {:?}: ", "Found invalid dot `.` at position {}."), - name, - index - )); - } - } - '?' | ',' | '\'' | '\"' | '\\' | '/' | ':' | ')' | '(' | '+' | '*' | '%' | '~' - | '\r' | '\n' | '\0' | '\u{0001}' | '\u{0002}' | '\u{0003}' | '\u{0004}' - | '\u{0005}' | '\u{0006}' | '\u{0007}' | '\u{0008}' | '\u{0009}' | '\u{000b}' - | '\u{000c}' | '\u{000e}' | '\u{000f}' | '\u{007f}' => { - return Err(error::fmt!( - InvalidName, - concat!( - "Bad string {:?}: ", - "Table names can't contain ", - "a {:?} character, which was found at ", - "byte position {}." - ), - name, - c, - index - )); - } - '\u{feff}' => { - // Reject unicode char 'ZERO WIDTH NO-BREAK SPACE', - // aka UTF-8 BOM if it appears anywhere in the string. - return Err(error::fmt!( - InvalidName, - concat!( - "Bad string {:?}: ", - "Table names can't contain ", - "a UTF-8 BOM character, which was found at ", - "byte position {}." - ), - name, - index - )); - } - _ => (), - } - prev = c; - } - - Ok(Self { name }) - } - - /// Construct a table name without validating it. - /// - /// This breaks API encapsulation and is only intended for use - /// when the the string was already previously validated. - /// - /// The QuestDB server will reject an invalid table name. - pub fn new_unchecked(name: &'a str) -> Self { - Self { name } - } -} - -/// A validated column name. -/// -/// This type simply wraps a `&str`. -/// -/// When you pass a `ColumnName` instead of a plain string to a [`Buffer`] method, -/// it doesn't have to validate it again. This saves CPU cycles. -#[derive(Clone, Copy)] -pub struct ColumnName<'a> { - name: &'a str, -} - -impl<'a> ColumnName<'a> { - /// Construct a validated table name. - pub fn new(name: &'a str) -> Result { - if name.is_empty() { - return Err(error::fmt!( - InvalidName, - "Column names must have a non-zero length." - )); - } - - for (index, c) in name.chars().enumerate() { - match c { - '?' | '.' | ',' | '\'' | '\"' | '\\' | '/' | ':' | ')' | '(' | '+' | '-' | '*' - | '%' | '~' | '\r' | '\n' | '\0' | '\u{0001}' | '\u{0002}' | '\u{0003}' - | '\u{0004}' | '\u{0005}' | '\u{0006}' | '\u{0007}' | '\u{0008}' | '\u{0009}' - | '\u{000b}' | '\u{000c}' | '\u{000e}' | '\u{000f}' | '\u{007f}' => { - return Err(error::fmt!( - InvalidName, - concat!( - "Bad string {:?}: ", - "Column names can't contain ", - "a {:?} character, which was found at ", - "byte position {}." - ), - name, - c, - index - )); - } - '\u{FEFF}' => { - // Reject unicode char 'ZERO WIDTH NO-BREAK SPACE', - // aka UTF-8 BOM if it appears anywhere in the string. - return Err(error::fmt!( - InvalidName, - concat!( - "Bad string {:?}: ", - "Column names can't contain ", - "a UTF-8 BOM character, which was found at ", - "byte position {}." - ), - name, - index - )); - } - _ => (), - } - } - - Ok(Self { name }) - } - - /// Construct a column name without validating it. - /// - /// This breaks API encapsulation and is only intended for use - /// when the the string was already previously validated. - /// - /// The QuestDB server will reject an invalid column name. - pub fn new_unchecked(name: &'a str) -> Self { - Self { name } - } -} - -impl<'a> TryFrom<&'a str> for TableName<'a> { - type Error = self::Error; - - fn try_from(name: &'a str) -> Result { - Self::new(name) - } -} - -impl<'a> TryFrom<&'a str> for ColumnName<'a> { - type Error = self::Error; - - fn try_from(name: &'a str) -> Result { - Self::new(name) - } -} - -impl From for Error { - fn from(_: Infallible) -> Self { - unreachable!() - } -} - -fn write_escaped_impl(check_escape_fn: C, quoting_fn: Q, output: &mut Vec, s: &str) -where - C: Fn(u8) -> bool, - Q: Fn(&mut Vec), -{ - let mut to_escape = 0usize; - for b in s.bytes() { - if check_escape_fn(b) { - to_escape += 1; - } - } - - quoting_fn(output); - - if to_escape == 0 { - // output.push_str(s); - output.extend_from_slice(s.as_bytes()); - } else { - let additional = s.len() + to_escape; - output.reserve(additional); - let mut index = output.len(); - unsafe { output.set_len(index + additional) }; - for b in s.bytes() { - if check_escape_fn(b) { - unsafe { - *output.get_unchecked_mut(index) = b'\\'; - } - index += 1; - } - - unsafe { - *output.get_unchecked_mut(index) = b; - } - index += 1; - } - } - - quoting_fn(output); -} - -fn must_escape_unquoted(c: u8) -> bool { - matches!(c, b' ' | b',' | b'=' | b'\n' | b'\r' | b'\\') -} - -fn must_escape_quoted(c: u8) -> bool { - matches!(c, b'\n' | b'\r' | b'"' | b'\\') -} - -fn write_escaped_unquoted(output: &mut Vec, s: &str) { - write_escaped_impl(must_escape_unquoted, |_output| (), output, s); -} - -fn write_escaped_quoted(output: &mut Vec, s: &str) { - write_escaped_impl(must_escape_quoted, |output| output.push(b'"'), output, s) -} - -enum Connection { - Direct(Socket), - Tls(Box>), -} - -impl Connection { - fn send_key_id(&mut self, key_id: &str) -> Result<()> { - writeln!(self, "{key_id}") - .map_err(|io_err| map_io_to_socket_err("Failed to send key_id: ", io_err))?; - Ok(()) - } - - fn read_challenge(&mut self) -> Result> { - let mut buf = Vec::new(); - let mut reader = BufReader::new(self); - reader.read_until(b'\n', &mut buf).map_err(|io_err| { - map_io_to_socket_err( - "Failed to read authentication challenge (timed out?): ", - io_err, - ) - })?; - if buf.last().copied().unwrap_or(b'\0') != b'\n' { - return Err(if buf.is_empty() { - error::fmt!( - AuthError, - concat!( - "Did not receive auth challenge. ", - "Is the database configured to require ", - "authentication?" - ) - ) - } else { - error::fmt!(AuthError, "Received incomplete auth challenge: {:?}", buf) - }); - } - buf.pop(); // b'\n' - Ok(buf) - } - - fn authenticate(&mut self, auth: &EcdsaAuthParams) -> Result<()> { - if auth.key_id.contains('\n') { - return Err(error::fmt!( - AuthError, - "Bad key id {:?}: Should not contain new-line char.", - auth.key_id - )); - } - let key_pair = parse_key_pair(auth)?; - self.send_key_id(auth.key_id.as_str())?; - let challenge = self.read_challenge()?; - let rng = SystemRandom::new(); - let signature = key_pair - .sign(&rng, &challenge[..]) - .map_err(|unspecified_err| { - error::fmt!(AuthError, "Failed to sign challenge: {}", unspecified_err) - })?; - let mut encoded_sig = Base64::encode_string(signature.as_ref()); - encoded_sig.push('\n'); - let buf = encoded_sig.as_bytes(); - if let Err(io_err) = self.write_all(buf) { - return Err(map_io_to_socket_err( - "Could not send signed challenge: ", - io_err, - )); - } - Ok(()) - } -} - -enum ProtocolHandler { - Socket(Connection), - - #[cfg(feature = "ilp-over-http")] - Http(HttpHandlerState), -} - -impl io::Read for Connection { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - match self { - Self::Direct(sock) => sock.read(buf), - Self::Tls(stream) => stream.read(buf), - } - } -} - -impl io::Write for Connection { - fn write(&mut self, buf: &[u8]) -> io::Result { - match self { - Self::Direct(sock) => sock.write(buf), - Self::Tls(stream) => stream.write(buf), - } - } - - fn flush(&mut self) -> io::Result<()> { - match self { - Self::Direct(sock) => sock.flush(), - Self::Tls(stream) => stream.flush(), - } - } -} - -#[derive(Debug, Copy, Clone, PartialEq)] -enum OpCase { - Init = Op::Table as isize, - TableWritten = Op::Symbol as isize | Op::Column as isize, - SymbolWritten = Op::Symbol as isize | Op::Column as isize | Op::At as isize, - ColumnWritten = Op::Column as isize | Op::At as isize, - MayFlushOrTable = Op::Flush as isize | Op::Table as isize, -} - -impl OpCase { - fn next_op_descr(self) -> &'static str { - match self { - OpCase::Init => "should have called `table` instead", - OpCase::TableWritten => "should have called `symbol` or `column` instead", - OpCase::SymbolWritten => "should have called `symbol`, `column` or `at` instead", - OpCase::ColumnWritten => "should have called `column` or `at` instead", - OpCase::MayFlushOrTable => "should have called `flush` or `table` instead", - } - } -} - -// IMPORTANT: This struct MUST remain `Copy` to ensure that -// there are no heap allocations when performing marker operations. -#[derive(Debug, Clone, Copy)] -struct BufferState { - op_case: OpCase, - row_count: usize, - first_table_len: Option, - transactional: bool, -} - -impl BufferState { - fn new() -> Self { - Self { - op_case: OpCase::Init, - row_count: 0, - first_table_len: None, - transactional: true, - } - } -} - -/// A reusable buffer to prepare a batch of ILP messages. -/// -/// # Example -/// -/// ```no_run -/// # use questdb::Result; -/// # use questdb::ingress::SenderBuilder; -/// -/// # fn main() -> Result<()> { -/// # let mut sender = SenderBuilder::from_conf("http::addr=localhost:9000;")?.build()?; -/// # use questdb::Result; -/// use questdb::ingress::{Buffer, TimestampMicros, TimestampNanos}; -/// let mut buffer = sender.new_buffer(); -/// -/// // first row -/// buffer -/// .table("table1")? -/// .symbol("bar", "baz")? -/// .column_bool("a", false)? -/// .column_i64("b", 42)? -/// .column_f64("c", 3.14)? -/// .column_str("d", "hello")? -/// .column_ts("e", TimestampMicros::now())? -/// .at(TimestampNanos::now())?; -/// -/// // second row -/// buffer -/// .table("table2")? -/// .symbol("foo", "bar")? -/// .at(TimestampNanos::now())?; -/// # Ok(()) -/// # } -/// ``` -/// -/// Send the buffer to QuestDB using [`sender.flush(&mut buffer)`](Sender::flush). -/// -/// # Sequential Coupling -/// The Buffer API is sequentially coupled: -/// * A row always starts with [`table`](Buffer::table). -/// * A row must contain at least one [`symbol`](Buffer::symbol) or -/// column ( -/// [`column_bool`](Buffer::column_bool), -/// [`column_i64`](Buffer::column_i64), -/// [`column_f64`](Buffer::column_f64), -/// [`column_str`](Buffer::column_str), -/// [`column_arr`](Buffer::column_arr), -/// [`column_ts`](Buffer::column_ts)). -/// * Symbols must appear before columns. -/// * A row must be terminated with either [`at`](Buffer::at) or -/// [`at_now`](Buffer::at_now). -/// -/// This diagram visualizes the sequence: -/// -/// -/// -/// # Buffer method calls, Serialized ILP types and QuestDB types -/// -/// | Buffer Method | Serialized as ILP type (Click on link to see possible casts) | -/// |---------------|--------------------------------------------------------------| -/// | [`symbol`](Buffer::symbol) | [`SYMBOL`](https://questdb.io/docs/concept/symbol/) | -/// | [`column_bool`](Buffer::column_bool) | [`BOOLEAN`](https://questdb.io/docs/reference/api/ilp/columnset-types#boolean) | -/// | [`column_i64`](Buffer::column_i64) | [`INTEGER`](https://questdb.io/docs/reference/api/ilp/columnset-types#integer) | -/// | [`column_f64`](Buffer::column_f64) | [`FLOAT`](https://questdb.io/docs/reference/api/ilp/columnset-types#float) | -/// | [`column_str`](Buffer::column_str) | [`STRING`](https://questdb.io/docs/reference/api/ilp/columnset-types#string) | -/// | [`column_arr`](Buffer::column_arr) | [`ARRAY`](https://questdb.io/docs/reference/api/ilp/columnset-types#array) | -/// | [`column_ts`](Buffer::column_ts) | [`TIMESTAMP`](https://questdb.io/docs/reference/api/ilp/columnset-types#timestamp) | -/// -/// QuestDB supports both `STRING` and `SYMBOL` column types. -/// -/// To understand the difference, refer to the -/// [QuestDB documentation](https://questdb.io/docs/concept/symbol/). In a nutshell, -/// symbols are interned strings, most suitable for identifiers that are repeated many -/// times throughout the column. They offer an advantage in storage space and query -/// performance. -/// -/// # Inserting NULL values -/// -/// To insert a NULL value, skip the symbol or column for that row. -/// -/// # Recovering from validation errors -/// -/// If you want to recover from potential validation errors, call -/// [`buffer.set_marker()`](Buffer::set_marker) to track the last known good state, -/// append as many rows or parts of rows as you like, and then call -/// [`buffer.clear_marker()`](Buffer::clear_marker) on success. -/// -/// If there was an error in one of the rows, use -/// [`buffer.rewind_to_marker()`](Buffer::rewind_to_marker) to go back to the -/// marked last known good state. -/// -#[derive(Debug, Clone)] -pub struct Buffer { - output: Vec, - state: BufferState, - marker: Option<(usize, BufferState)>, - max_name_len: usize, - version: ProtocolVersion, -} - -impl Buffer { - /// Creates a new [`Buffer`] with default parameters. - /// - /// - Uses the specified protocol version - /// - Sets maximum name length to **127 characters** (QuestDB server default) - /// - /// This is equivalent to [`Sender::new_buffer`] when using the [`Sender::protocol_version`] - /// and [`Sender::max_name_len`] is 127. - /// - /// For custom name lengths, use [`Self::with_max_name_len`] - pub fn new(protocol_version: ProtocolVersion) -> Self { - Self::with_max_name_len(protocol_version, MAX_NAME_LEN_DEFAULT) - } - - /// Creates a new [`Buffer`] with a custom maximum name length. - /// - /// - `max_name_len`: Maximum allowed length for table/column names, match - /// your QuestDB server's `cairo.max.file.name.length` configuration - /// - `protocol_version`: Protocol version to use - /// - /// This is equivalent to [`Sender::new_buffer`] when using the [`Sender::protocol_version`] - /// and [`Sender::max_name_len`]. - /// - /// For the default max name length limit (127), use [`Self::new`]. - pub fn with_max_name_len(protocol_version: ProtocolVersion, max_name_len: usize) -> Self { - Self { - output: Vec::new(), - state: BufferState::new(), - marker: None, - max_name_len, - version: protocol_version, - } - } - - /// Pre-allocate to ensure the buffer has enough capacity for at least the - /// specified additional byte count. This may be rounded up. - /// This does not allocate if such additional capacity is already satisfied. - /// See: `capacity`. - pub fn reserve(&mut self, additional: usize) { - self.output.reserve(additional); - } - - /// The number of bytes accumulated in the buffer. - pub fn len(&self) -> usize { - self.output.len() - } - - /// The number of rows accumulated in the buffer. - pub fn row_count(&self) -> usize { - self.state.row_count - } - - /// Tells whether the buffer is transactional. It is transactional iff it contains - /// data for at most one table. Additionally, you must send the buffer over HTTP to - /// get transactional behavior. - pub fn transactional(&self) -> bool { - self.state.transactional - } - - pub fn is_empty(&self) -> bool { - self.output.is_empty() - } - - /// The total number of bytes the buffer can hold before it needs to resize. - pub fn capacity(&self) -> usize { - self.output.capacity() - } - - pub fn as_bytes(&self) -> &[u8] { - &self.output - } - - /// Mark a rewind point. - /// This allows undoing accumulated changes to the buffer for one or more - /// rows by calling [`rewind_to_marker`](Buffer::rewind_to_marker). - /// Any previous marker will be discarded. - /// Once the marker is no longer needed, call - /// [`clear_marker`](Buffer::clear_marker). - pub fn set_marker(&mut self) -> Result<()> { - if (self.state.op_case as isize & Op::Table as isize) == 0 { - return Err(error::fmt!( - InvalidApiCall, - concat!( - "Can't set the marker whilst constructing a line. ", - "A marker may only be set on an empty buffer or after ", - "`at` or `at_now` is called." - ) - )); - } - self.marker = Some((self.output.len(), self.state)); - Ok(()) - } - - /// Undo all changes since the last [`set_marker`](Buffer::set_marker) - /// call. - /// - /// As a side-effect, this also clears the marker. - pub fn rewind_to_marker(&mut self) -> Result<()> { - if let Some((position, state)) = self.marker.take() { - self.output.truncate(position); - self.state = state; - Ok(()) - } else { - Err(error::fmt!( - InvalidApiCall, - "Can't rewind to the marker: No marker set." - )) - } - } - - /// Discard any marker as may have been set by - /// [`set_marker`](Buffer::set_marker). - /// - /// Idempotent. - pub fn clear_marker(&mut self) { - self.marker = None; - } - - /// Reset the buffer and clear contents whilst retaining - /// [`capacity`](Buffer::capacity). - pub fn clear(&mut self) { - self.output.clear(); - self.state = BufferState::new(); - self.marker = None; - } - - /// Check if the next API operation is allowed as per the OP case state machine. - #[inline(always)] - fn check_op(&self, op: Op) -> Result<()> { - if (self.state.op_case as isize & op as isize) > 0 { - Ok(()) - } else { - Err(error::fmt!( - InvalidApiCall, - "State error: Bad call to `{}`, {}.", - op.descr(), - self.state.op_case.next_op_descr() - )) - } - } - - /// Checks if this buffer is ready to be flushed to a sender via one of the - /// [`Sender::flush`] functions. An [`Ok`] value indicates that the buffer - /// is ready to be flushed via a [`Sender`] while an [`Err`] will contain a - /// message indicating why this [`Buffer`] cannot be flushed at the moment. - #[inline(always)] - pub fn check_can_flush(&self) -> Result<()> { - self.check_op(Op::Flush) - } - - #[inline(always)] - fn validate_max_name_len(&self, name: &str) -> Result<()> { - if name.len() > self.max_name_len { - return Err(error::fmt!( - InvalidName, - "Bad name: {:?}: Too long (max {} characters)", - name, - self.max_name_len - )); - } - Ok(()) - } - - /// Begin recording a new row for the given table. - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// buffer.table("table_name")?; - /// # Ok(()) - /// # } - /// ``` - /// - /// or - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// use questdb::ingress::TableName; - /// - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// let table_name = TableName::new("table_name")?; - /// buffer.table(table_name)?; - /// # Ok(()) - /// # } - /// ``` - pub fn table<'a, N>(&mut self, name: N) -> Result<&mut Self> - where - N: TryInto>, - Error: From, - { - let name: TableName<'a> = name.try_into()?; - self.validate_max_name_len(name.name)?; - self.check_op(Op::Table)?; - let table_begin = self.output.len(); - write_escaped_unquoted(&mut self.output, name.name); - let table_end = self.output.len(); - self.state.op_case = OpCase::TableWritten; - - // A buffer stops being transactional if it targets multiple tables. - if let Some(first_table_len) = &self.state.first_table_len { - let first_table = &self.output[0..(first_table_len.get())]; - let this_table = &self.output[table_begin..table_end]; - if first_table != this_table { - self.state.transactional = false; - } - } else { - debug_assert!(table_begin == 0); - - // This is a bit confusing, so worth explaining: - // `NonZeroUsize::new(table_end)` will return `None` if `table_end` is 0, - // but we know that `table_end` is never 0 here, we just need an option type - // anyway, so we don't bother unwrapping it to then wrap it again. - let first_table_len = NonZeroUsize::new(table_end); - - // Instead we just assert that it's `Some`. - debug_assert!(first_table_len.is_some()); - - self.state.first_table_len = first_table_len; - } - Ok(self) - } - - /// Record a symbol for the given column. - /// Make sure you record all symbol columns before any other column type. - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// buffer.symbol("col_name", "value")?; - /// # Ok(()) - /// # } - /// ``` - /// - /// or - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// let value: String = "value".to_owned(); - /// buffer.symbol("col_name", value)?; - /// # Ok(()) - /// # } - /// ``` - /// - /// or - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// use questdb::ingress::ColumnName; - /// - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// let col_name = ColumnName::new("col_name")?; - /// buffer.symbol(col_name, "value")?; - /// # Ok(()) - /// # } - /// ``` - /// - pub fn symbol<'a, N, S>(&mut self, name: N, value: S) -> Result<&mut Self> - where - N: TryInto>, - S: AsRef, - Error: From, - { - let name: ColumnName<'a> = name.try_into()?; - self.validate_max_name_len(name.name)?; - self.check_op(Op::Symbol)?; - self.output.push(b','); - write_escaped_unquoted(&mut self.output, name.name); - self.output.push(b'='); - write_escaped_unquoted(&mut self.output, value.as_ref()); - self.state.op_case = OpCase::SymbolWritten; - Ok(self) - } - - fn write_column_key<'a, N>(&mut self, name: N) -> Result<&mut Self> - where - N: TryInto>, - Error: From, - { - let name: ColumnName<'a> = name.try_into()?; - self.validate_max_name_len(name.name)?; - self.check_op(Op::Column)?; - self.output - .push(if (self.state.op_case as isize & Op::Symbol as isize) > 0 { - b' ' - } else { - b',' - }); - write_escaped_unquoted(&mut self.output, name.name); - self.output.push(b'='); - self.state.op_case = OpCase::ColumnWritten; - Ok(self) - } - - /// Record a boolean value for the given column. - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// buffer.column_bool("col_name", true)?; - /// # Ok(()) - /// # } - /// ``` - /// - /// or - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// use questdb::ingress::ColumnName; - /// - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// let col_name = ColumnName::new("col_name")?; - /// buffer.column_bool(col_name, true)?; - /// # Ok(()) - /// # } - /// ``` - pub fn column_bool<'a, N>(&mut self, name: N, value: bool) -> Result<&mut Self> - where - N: TryInto>, - Error: From, - { - self.write_column_key(name)?; - self.output.push(if value { b't' } else { b'f' }); - Ok(self) - } - - /// Record an integer value for the given column. - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// buffer.column_i64("col_name", 42)?; - /// # Ok(()) - /// # } - /// ``` - /// - /// or - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// use questdb::ingress::ColumnName; - /// - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// let col_name = ColumnName::new("col_name")?; - /// buffer.column_i64(col_name, 42); - /// # Ok(()) - /// # } - /// ``` - pub fn column_i64<'a, N>(&mut self, name: N, value: i64) -> Result<&mut Self> - where - N: TryInto>, - Error: From, - { - self.write_column_key(name)?; - let mut buf = itoa::Buffer::new(); - let printed = buf.format(value); - self.output.extend_from_slice(printed.as_bytes()); - self.output.push(b'i'); - Ok(self) - } - - /// Record a floating point value for the given column. - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// buffer.column_f64("col_name", 3.14)?; - /// # Ok(()) - /// # } - /// ``` - /// - /// or - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// use questdb::ingress::ColumnName; - /// - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// let col_name = ColumnName::new("col_name")?; - /// buffer.column_f64(col_name, 3.14)?; - /// # Ok(()) - /// # } - /// ``` - pub fn column_f64<'a, N>(&mut self, name: N, value: f64) -> Result<&mut Self> - where - N: TryInto>, - Error: From, - { - self.write_column_key(name)?; - if !matches!(self.version, ProtocolVersion::V1) { - self.output.push(b'='); - self.output.push(DOUBLE_BINARY_FORMAT_TYPE); - self.output.extend_from_slice(&value.to_le_bytes()) - } else { - let mut ser = F64Serializer::new(value); - self.output.extend_from_slice(ser.as_str().as_bytes()) - } - Ok(self) - } - - /// Record a string value for the given column. - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// buffer.column_str("col_name", "value")?; - /// # Ok(()) - /// # } - /// ``` - /// - /// or - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// let value: String = "value".to_owned(); - /// buffer.column_str("col_name", value)?; - /// # Ok(()) - /// # } - /// ``` - /// - /// or - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// use questdb::ingress::ColumnName; - /// - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// let col_name = ColumnName::new("col_name")?; - /// buffer.column_str(col_name, "value")?; - /// # Ok(()) - /// # } - /// ``` - pub fn column_str<'a, N, S>(&mut self, name: N, value: S) -> Result<&mut Self> - where - N: TryInto>, - S: AsRef, - Error: From, - { - self.write_column_key(name)?; - write_escaped_quoted(&mut self.output, value.as_ref()); - Ok(self) - } - - /// Record a multidimensional array value for the given column. - /// - /// Supports arrays with up to [`MAX_ARRAY_DIMS`] dimensions. The array elements must - /// be of type `f64`, which is currently the only supported data type. - /// - /// **Note**: QuestDB server version 9.0.0 or later is required for array support. - /// - /// # Examples - /// - /// Recording a 2D array using slices: - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// let array_2d = vec![vec![1.1, 2.2], vec![3.3, 4.4]]; - /// buffer.column_arr("array_col", &array_2d)?; - /// # Ok(()) - /// # } - /// ``` - /// - /// Recording a 3D array using vectors: - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, ColumnName, SenderBuilder}; - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x1")?; - /// let array_3d = vec![vec![vec![42.0; 4]; 3]; 2]; - /// let col_name = ColumnName::new("col1")?; - /// buffer.column_arr(col_name, &array_3d)?; - /// # Ok(()) - /// # } - /// ``` - /// - /// # Errors - /// - /// Returns [`Error`] if: - /// - Array dimensions exceed [`MAX_ARRAY_DIMS`] - /// - Failed to get dimension sizes - /// - Column name validation fails - /// - Protocol version v1 is used (arrays require v2+) - #[allow(private_bounds)] - pub fn column_arr<'a, N, T, D>(&mut self, name: N, view: &T) -> Result<&mut Self> - where - N: TryInto>, - T: NdArrayView, - D: ArrayElement + ArrayElementSealed, - Error: From, - { - if self.version == ProtocolVersion::V1 { - return Err(error::fmt!( - ProtocolVersionError, - "Protocol version v1 does not support array datatype", - )); - } - let ndim = view.ndim(); - if ndim == 0 { - return Err(error::fmt!( - ArrayError, - "Zero-dimensional arrays are not supported", - )); - } - - // check dimension less equal than max dims - if MAX_ARRAY_DIMS < ndim { - return Err(error::fmt!( - ArrayError, - "Array dimension mismatch: expected at most {} dimensions, but got {}", - MAX_ARRAY_DIMS, - ndim - )); - } - - let array_buf_size = check_and_get_array_bytes_size(view)?; - self.write_column_key(name)?; - // binary format flag '=' - self.output.push(b'='); - // binary format entity type - self.output.push(ARRAY_BINARY_FORMAT_TYPE); - // ndarr datatype - self.output.push(D::type_tag()); - // ndarr dims - self.output.push(ndim as u8); - - let dim_header_size = size_of::() * ndim; - self.output.reserve(dim_header_size + array_buf_size); - - for i in 0..ndim { - // ndarr shape - self.output - .extend_from_slice((view.dim(i)? as u32).to_le_bytes().as_slice()); - } - - let index = self.output.len(); - let writeable = - unsafe { from_raw_parts_mut(self.output.as_mut_ptr().add(index), array_buf_size) }; - - // ndarr data - ndarr::write_array_data(view, writeable, array_buf_size)?; - unsafe { self.output.set_len(array_buf_size + index) } - Ok(self) - } - - /// Record a timestamp value for the given column. - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// use questdb::ingress::TimestampMicros; - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// buffer.column_ts("col_name", TimestampMicros::now())?; - /// # Ok(()) - /// # } - /// ``` - /// - /// or - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// use questdb::ingress::TimestampMicros; - /// - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// buffer.column_ts("col_name", TimestampMicros::new(1659548204354448))?; - /// # Ok(()) - /// # } - /// ``` - /// - /// or - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// use questdb::ingress::TimestampMicros; - /// use questdb::ingress::ColumnName; - /// - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?; - /// let col_name = ColumnName::new("col_name")?; - /// buffer.column_ts(col_name, TimestampMicros::now())?; - /// # Ok(()) - /// # } - /// ``` - /// - /// or you can also pass in a `TimestampNanos`. - /// - /// Note that both `TimestampMicros` and `TimestampNanos` can be constructed - /// easily from either `std::time::SystemTime` or `chrono::DateTime`. - /// - /// This last option requires the `chrono_timestamp` feature. - pub fn column_ts<'a, N, T>(&mut self, name: N, value: T) -> Result<&mut Self> - where - N: TryInto>, - T: TryInto, - Error: From, - Error: From, - { - self.write_column_key(name)?; - let timestamp: Timestamp = value.try_into()?; - let timestamp: TimestampMicros = timestamp.try_into()?; - let mut buf = itoa::Buffer::new(); - let printed = buf.format(timestamp.as_i64()); - self.output.extend_from_slice(printed.as_bytes()); - self.output.push(b't'); - Ok(self) - } - - /// Complete the current row with the designated timestamp. After this call, you can - /// start recording the next row by calling [Buffer::table] again, or you can send - /// the accumulated batch by calling [Sender::flush] or one of its variants. - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// use questdb::ingress::TimestampNanos; - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?.symbol("a", "b")?; - /// buffer.at(TimestampNanos::now())?; - /// # Ok(()) - /// # } - /// ``` - /// - /// or - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// use questdb::ingress::TimestampNanos; - /// - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?.symbol("a", "b")?; - /// buffer.at(TimestampNanos::new(1659548315647406592))?; - /// # Ok(()) - /// # } - /// ``` - /// - /// You can also pass in a `TimestampMicros`. - /// - /// Note that both `TimestampMicros` and `TimestampNanos` can be constructed - /// easily from either `std::time::SystemTime` or `chrono::DateTime`. - /// - pub fn at(&mut self, timestamp: T) -> Result<()> - where - T: TryInto, - Error: From, - { - self.check_op(Op::At)?; - let timestamp: Timestamp = timestamp.try_into()?; - - // https://github.com/rust-lang/rust/issues/115880 - let timestamp: Result = timestamp.try_into(); - let timestamp: TimestampNanos = timestamp?; - - let epoch_nanos = timestamp.as_i64(); - if epoch_nanos < 0 { - return Err(error::fmt!( - InvalidTimestamp, - "Timestamp {} is negative. It must be >= 0.", - epoch_nanos - )); - } - let mut buf = itoa::Buffer::new(); - let printed = buf.format(epoch_nanos); - self.output.push(b' '); - self.output.extend_from_slice(printed.as_bytes()); - self.output.push(b'\n'); - self.state.op_case = OpCase::MayFlushOrTable; - self.state.row_count += 1; - Ok(()) - } - - /// Complete the current row without providing a timestamp. The QuestDB instance - /// will insert its own timestamp. - /// - /// Letting the server assign the timestamp can be faster since it reliably avoids - /// out-of-order operations in the database for maximum ingestion throughput. However, - /// it removes the ability to deduplicate rows. - /// - /// This is NOT equivalent to calling [Buffer::at] with the current time: the QuestDB - /// server will set the timestamp only after receiving the row. If you're flushing - /// infrequently, the server-assigned timestamp may be significantly behind the - /// time the data was recorded in the buffer. - /// - /// In almost all cases, you should prefer the [Buffer::at] function. - /// - /// After this call, you can start recording the next row by calling [Buffer::table] - /// again, or you can send the accumulated batch by calling [Sender::flush] or one of - /// its variants. - /// - /// ```no_run - /// # use questdb::Result; - /// # use questdb::ingress::{Buffer, SenderBuilder}; - /// # fn main() -> Result<()> { - /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; - /// # let mut buffer = sender.new_buffer(); - /// # buffer.table("x")?.symbol("a", "b")?; - /// buffer.at_now()?; - /// # Ok(()) - /// # } - /// ``` - pub fn at_now(&mut self) -> Result<()> { - self.check_op(Op::At)?; - self.output.push(b'\n'); - self.state.op_case = OpCase::MayFlushOrTable; - self.state.row_count += 1; - Ok(()) - } -} - -/// Connects to a QuestDB instance and inserts data via the ILP protocol. -/// -/// * To construct an instance, use [`Sender::from_conf`] or the [`SenderBuilder`]. -/// * To prepare messages, use [`Buffer`] objects. -/// * To send messages, call the [`flush`](Sender::flush) method. -pub struct Sender { - descr: String, - handler: ProtocolHandler, - connected: bool, - max_buf_size: usize, - protocol_version: ProtocolVersion, - max_name_len: usize, -} - -impl std::fmt::Debug for Sender { - fn fmt(&self, f: &mut Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { - f.write_str(self.descr.as_str()) - } -} - -#[derive(PartialEq, Debug, Clone)] -struct EcdsaAuthParams { - key_id: String, - priv_key: String, - pub_key_x: String, - pub_key_y: String, + } } -#[derive(PartialEq, Debug, Clone)] -enum AuthParams { - Ecdsa(EcdsaAuthParams), - - #[cfg(feature = "ilp-over-http")] - Basic(BasicAuthParams), - - #[cfg(feature = "ilp-over-http")] - Token(TokenAuthParams), +#[cfg(feature = "_sender-tcp")] +fn map_io_to_socket_err(prefix: &str, io_err: std::io::Error) -> error::Error { + fmt!(SocketError, "{}{}", prefix, io_err) } /// Possible sources of the root certificates used to validate the server's TLS @@ -1476,199 +161,6 @@ impl From for Port { } } -#[cfg(feature = "insecure-skip-verify")] -mod danger { - use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; - use rustls::{DigitallySignedStruct, Error, SignatureScheme}; - use rustls_pki_types::{CertificateDer, ServerName, UnixTime}; - - #[derive(Debug)] - pub struct NoCertificateVerification {} - - impl ServerCertVerifier for NoCertificateVerification { - fn verify_server_cert( - &self, - _end_entity: &CertificateDer<'_>, - _intermediates: &[CertificateDer<'_>], - _server_name: &ServerName<'_>, - _ocsp_response: &[u8], - _now: UnixTime, - ) -> Result { - Ok(ServerCertVerified::assertion()) - } - - fn verify_tls12_signature( - &self, - _message: &[u8], - _cert: &CertificateDer<'_>, - _dss: &DigitallySignedStruct, - ) -> Result { - Ok(HandshakeSignatureValid::assertion()) - } - - fn verify_tls13_signature( - &self, - _message: &[u8], - _cert: &CertificateDer<'_>, - _dss: &DigitallySignedStruct, - ) -> Result { - Ok(HandshakeSignatureValid::assertion()) - } - - #[cfg(feature = "aws-lc-crypto")] - fn supported_verify_schemes(&self) -> Vec { - rustls::crypto::aws_lc_rs::default_provider() - .signature_verification_algorithms - .supported_schemes() - } - - #[cfg(feature = "ring-crypto")] - fn supported_verify_schemes(&self) -> Vec { - rustls::crypto::ring::default_provider() - .signature_verification_algorithms - .supported_schemes() - } - } -} - -#[cfg(feature = "tls-webpki-certs")] -fn add_webpki_roots(root_store: &mut RootCertStore) { - root_store - .roots - .extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()) -} - -#[cfg(feature = "tls-native-certs")] -fn unpack_os_native_certs( - res: rustls_native_certs::CertificateResult, -) -> Result>> { - if !res.errors.is_empty() { - return Err(error::fmt!( - TlsError, - "Could not load OS native TLS certificates: {}", - res.errors - .iter() - .map(|e| e.to_string()) - .collect::>() - .join(", ") - )); - } - - Ok(res.certs) -} - -#[cfg(feature = "tls-native-certs")] -fn add_os_roots(root_store: &mut RootCertStore) -> Result<()> { - let os_certs = unpack_os_native_certs(rustls_native_certs::load_native_certs())?; - - let (valid_count, invalid_count) = root_store.add_parsable_certificates(os_certs); - if valid_count == 0 && invalid_count > 0 { - return Err(error::fmt!( - TlsError, - "No valid certificates found in native root store ({} found but were invalid)", - invalid_count - )); - } - Ok(()) -} - -fn configure_tls( - tls_enabled: bool, - tls_verify: bool, - tls_ca: CertificateAuthority, - tls_roots: &Option, -) -> Result>> { - if !tls_enabled { - return Ok(None); - } - - let mut root_store = RootCertStore::empty(); - if tls_verify { - match (tls_ca, tls_roots) { - #[cfg(feature = "tls-webpki-certs")] - (CertificateAuthority::WebpkiRoots, None) => { - add_webpki_roots(&mut root_store); - } - - #[cfg(feature = "tls-webpki-certs")] - (CertificateAuthority::WebpkiRoots, Some(_)) => { - return Err(error::fmt!(ConfigError, "Config parameter \"tls_roots\" must be unset when \"tls_ca\" is set to \"webpki_roots\".")); - } - - #[cfg(feature = "tls-native-certs")] - (CertificateAuthority::OsRoots, None) => { - add_os_roots(&mut root_store)?; - } - - #[cfg(feature = "tls-native-certs")] - (CertificateAuthority::OsRoots, Some(_)) => { - return Err(error::fmt!(ConfigError, "Config parameter \"tls_roots\" must be unset when \"tls_ca\" is set to \"os_roots\".")); - } - - #[cfg(all(feature = "tls-webpki-certs", feature = "tls-native-certs"))] - (CertificateAuthority::WebpkiAndOsRoots, None) => { - add_webpki_roots(&mut root_store); - add_os_roots(&mut root_store)?; - } - - #[cfg(all(feature = "tls-webpki-certs", feature = "tls-native-certs"))] - (CertificateAuthority::WebpkiAndOsRoots, Some(_)) => { - return Err(error::fmt!(ConfigError, "Config parameter \"tls_roots\" must be unset when \"tls_ca\" is set to \"webpki_and_os_roots\".")); - } - - (CertificateAuthority::PemFile, Some(ca_file)) => { - let certfile = std::fs::File::open(ca_file).map_err(|io_err| { - error::fmt!( - TlsError, - concat!( - "Could not open tls_roots certificate authority ", - "file from path {:?}: {}" - ), - ca_file, - io_err - ) - })?; - let mut reader = BufReader::new(certfile); - let der_certs = rustls_pemfile::certs(&mut reader) - .collect::, _>>() - .map_err(|io_err| { - error::fmt!( - TlsError, - concat!( - "Could not read certificate authority ", - "file from path {:?}: {}" - ), - ca_file, - io_err - ) - })?; - root_store.add_parsable_certificates(der_certs); - } - - (CertificateAuthority::PemFile, None) => { - return Err(error::fmt!(ConfigError, "Config parameter \"tls_roots\" is required when \"tls_ca\" is set to \"pem_file\".")); - } - } - } - - let mut config = rustls::ClientConfig::builder() - .with_root_certificates(root_store) - .with_no_client_auth(); - - // TLS log file for debugging. - // Set the SSLKEYLOGFILE env variable to a writable location. - config.key_log = Arc::new(rustls::KeyLogFile::new()); - - #[cfg(feature = "insecure-skip-verify")] - if !tls_verify { - config - .dangerous() - .set_certificate_verifier(Arc::new(danger::NoCertificateVerification {})); - } - - Ok(Some(Arc::new(config))) -} - fn validate_auto_flush_params(params: &HashMap) -> Result<()> { if let Some(auto_flush) = params.get("auto_flush") { if auto_flush.as_str() != "off" { @@ -1695,18 +187,20 @@ fn validate_auto_flush_params(params: &HashMap) -> Result<()> { /// Protocol used to communicate with the QuestDB server. #[derive(PartialEq, Debug, Clone, Copy)] pub enum Protocol { + #[cfg(feature = "_sender-tcp")] /// ILP over TCP (streaming). Tcp, + #[cfg(feature = "_sender-tcp")] /// TCP + TLS Tcps, - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "_sender-http")] /// ILP over HTTP (request-response) /// Version 1 is compatible with the InfluxDB Line Protocol. Http, - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "_sender-http")] /// HTTP + TLS Https, } @@ -1720,62 +214,66 @@ impl Display for Protocol { impl Protocol { fn default_port(&self) -> &str { match self { + #[cfg(feature = "_sender-tcp")] Protocol::Tcp | Protocol::Tcps => "9009", - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "_sender-http")] Protocol::Http | Protocol::Https => "9000", } } fn tls_enabled(&self) -> bool { match self { + #[cfg(feature = "_sender-tcp")] Protocol::Tcp => false, + #[cfg(feature = "_sender-tcp")] Protocol::Tcps => true, - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "_sender-http")] Protocol::Http => false, - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "_sender-http")] Protocol::Https => true, } } + #[cfg(feature = "_sender-tcp")] fn is_tcpx(&self) -> bool { match self { - Protocol::Tcp => true, - Protocol::Tcps => true, - #[cfg(feature = "ilp-over-http")] - Protocol::Http => false, - #[cfg(feature = "ilp-over-http")] - Protocol::Https => false, + Protocol::Tcp | Protocol::Tcps => true, + #[cfg(feature = "_sender-http")] + Protocol::Http | Protocol::Https => false, } } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "_sender-http")] fn is_httpx(&self) -> bool { match self { - Protocol::Tcp => false, - Protocol::Tcps => false, - Protocol::Http => true, - Protocol::Https => true, + #[cfg(feature = "_sender-tcp")] + Protocol::Tcp | Protocol::Tcps => false, + Protocol::Http | Protocol::Https => true, } } fn schema(&self) -> &str { match self { + #[cfg(feature = "_sender-tcp")] Protocol::Tcp => "tcp", + #[cfg(feature = "_sender-tcp")] Protocol::Tcps => "tcps", - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "_sender-http")] Protocol::Http => "http", - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "_sender-http")] Protocol::Https => "https", } } fn from_schema(schema: &str) -> Result { match schema { + #[cfg(feature = "_sender-tcp")] "tcp" => Ok(Protocol::Tcp), + #[cfg(feature = "_sender-tcp")] "tcps" => Ok(Protocol::Tcps), - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "_sender-http")] "http" => Ok(Protocol::Http), - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "_sender-http")] "https" => Ok(Protocol::Https), _ => Err(error::fmt!(ConfigError, "Unsupported protocol: {}", schema)), } @@ -1784,54 +282,49 @@ impl Protocol { /// Accumulates parameters for a new `Sender` instance. /// -/// You can also create the builder from a config string or the `QDB_CLIENT_CONF` -/// environment variable. -/// -#[cfg_attr( - feature = "ilp-over-http", - doc = r##" -```no_run -# use questdb::Result; -use questdb::ingress::{Protocol, SenderBuilder}; -# fn main() -> Result<()> { -let mut sender = SenderBuilder::new(Protocol::Http, "localhost", 9009).build()?; -# Ok(()) -# } -``` -"## -)] +/// You can also create the builder from a config string. /// /// ```no_run /// # use questdb::Result; -/// use questdb::ingress::{Protocol, SenderBuilder}; +/// use questdb::ingress::SenderBuilder; /// /// # fn main() -> Result<()> { -/// let mut sender = SenderBuilder::new(Protocol::Tcp, "localhost", 9009).build()?; +/// let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; /// # Ok(()) /// # } /// ``` /// +/// Or create it from the `QDB_CLIENT_CONF` environment variable. +/// /// ```no_run /// # use questdb::Result; /// use questdb::ingress::SenderBuilder; /// /// # fn main() -> Result<()> { -/// let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; +/// // export QDB_CLIENT_CONF="https::addr=localhost:9000;" +/// let mut sender = SenderBuilder::from_env()?.build()?; /// # Ok(()) /// # } /// ``` /// +/// The `SenderBuilder` can also be built programmatically. +/// The minimum required parameters are the protocol, host, and port. +/// /// ```no_run /// # use questdb::Result; /// use questdb::ingress::SenderBuilder; +/// use questdb::ingress::Protocol; /// /// # fn main() -> Result<()> { -/// // export QDB_CLIENT_CONF="https::addr=localhost:9000;" -/// let mut sender = SenderBuilder::from_env()?.build()?; +/// # #[cfg(feature = "sync-sender-http")] { +/// let mut sender = SenderBuilder::new(Protocol::Http, "localhost", 9000).build()?; +/// # } +/// # #[cfg(all(not(feature = "sync-sender-http"), feature = "sync-sender-tcp"))] { +/// let mut sender = SenderBuilder::new(Protocol::Tcp, "localhost", 9009).build()?; +/// # } /// # Ok(()) /// # } /// ``` -/// #[derive(Debug, Clone)] pub struct SenderBuilder { protocol: Protocol, @@ -1844,8 +337,13 @@ pub struct SenderBuilder { username: ConfigSetting>, password: ConfigSetting>, token: ConfigSetting>, + + #[cfg(feature = "_sender-tcp")] token_x: ConfigSetting>, + + #[cfg(feature = "_sender-tcp")] token_y: ConfigSetting>, + protocol_version: ConfigSetting>, #[cfg(feature = "insecure-skip-verify")] @@ -1854,8 +352,8 @@ pub struct SenderBuilder { tls_ca: ConfigSetting, tls_roots: ConfigSetting>, - #[cfg(feature = "ilp-over-http")] - http: Option, + #[cfg(feature = "_sender-http")] + http: Option, } impl SenderBuilder { @@ -1949,7 +447,7 @@ impl SenderBuilder { "on" => true, "unsafe_off" => false, _ => { - return Err(error::fmt!( + return Err(fmt!( ConfigError, r##"Config parameter "tls_verify" must be either "on" or "unsafe_off".'"##, )) @@ -1959,7 +457,7 @@ impl SenderBuilder { #[cfg(not(feature = "insecure-skip-verify"))] { if !verify { - return Err(error::fmt!( + return Err(fmt!( ConfigError, r##"The "insecure-skip-verify" feature is not enabled, so "tls_verify=unsafe_off" is not supported"##, )); @@ -2015,17 +513,17 @@ impl SenderBuilder { )) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] "request_min_throughput" => { builder.request_min_throughput(parse_conf_value(key, val)?)? } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] "request_timeout" => { builder.request_timeout(Duration::from_millis(parse_conf_value(key, val)?))? } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] "retry_timeout" => { builder.retry_timeout(Duration::from_millis(parse_conf_value(key, val)?))? } @@ -2060,8 +558,14 @@ impl SenderBuilder { /// use questdb::ingress::{Protocol, SenderBuilder}; /// /// # fn main() -> Result<()> { + /// # #[cfg(feature = "sync-sender-tcp")] { /// let mut sender = SenderBuilder::new( /// Protocol::Tcp, "localhost", 9009).build()?; + /// # } + /// # #[cfg(all(not(feature = "sync-sender-tcp"), feature = "sync-sender-http"))] { + /// let mut sender = SenderBuilder::new( + /// Protocol::Http, "localhost", 9000).build()?; + /// # } /// # Ok(()) /// # } /// ``` @@ -2090,8 +594,13 @@ impl SenderBuilder { username: ConfigSetting::new_default(None), password: ConfigSetting::new_default(None), token: ConfigSetting::new_default(None), + + #[cfg(feature = "_sender-tcp")] token_x: ConfigSetting::new_default(None), + + #[cfg(feature = "_sender-tcp")] token_y: ConfigSetting::new_default(None), + protocol_version: ConfigSetting::new_default(None), #[cfg(feature = "insecure-skip-verify")] @@ -2100,9 +609,9 @@ impl SenderBuilder { tls_ca: ConfigSetting::new_default(tls_ca), tls_roots: ConfigSetting::new_default(None), - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] http: if protocol.is_httpx() { - Some(HttpConfig::default()) + Some(conf::HttpConfig::default()) } else { None }, @@ -2114,11 +623,25 @@ impl SenderBuilder { /// This may be relevant if your machine has multiple network interfaces. /// /// The default is `"0.0.0.0"`. - pub fn bind_interface>(mut self, addr: I) -> Result { - self.ensure_is_tcpx("bind_interface")?; - self.net_interface - .set_specified("bind_interface", Some(validate_value(addr.into())?))?; - Ok(self) + pub fn bind_interface>(self, addr: I) -> Result { + #[cfg(feature = "_sender-tcp")] + { + let mut builder = self; + builder.ensure_is_tcpx("bind_interface")?; + builder + .net_interface + .set_specified("bind_interface", Some(validate_value(addr.into())?))?; + Ok(builder) + } + + #[cfg(not(feature = "_sender-tcp"))] + { + let _ = addr; + Err(error::fmt!( + ConfigError, + "The \"bind_interface\" setting can only be used with the TCP protocol." + )) + } } /// Set the username for authentication. @@ -2152,17 +675,45 @@ impl SenderBuilder { } /// Set the ECDSA public key X for TCP authentication. - pub fn token_x(mut self, token_x: &str) -> Result { - self.token_x - .set_specified("token_x", Some(validate_value(token_x.to_string())?))?; - Ok(self) + pub fn token_x(self, token_x: &str) -> Result { + #[cfg(feature = "_sender-tcp")] + { + let mut builder = self; + builder + .token_x + .set_specified("token_x", Some(validate_value(token_x.to_string())?))?; + Ok(builder) + } + + #[cfg(not(feature = "_sender-tcp"))] + { + let _ = token_x; + Err(error::fmt!( + ConfigError, + "cannot specify \"token_x\": ECDSA authentication is only available with ILP/TCP and not available with ILP/HTTP." + )) + } } /// Set the ECDSA public key Y for TCP authentication. - pub fn token_y(mut self, token_y: &str) -> Result { - self.token_y - .set_specified("token_y", Some(validate_value(token_y.to_string())?))?; - Ok(self) + pub fn token_y(self, token_y: &str) -> Result { + #[cfg(feature = "_sender-tcp")] + { + let mut builder = self; + builder + .token_y + .set_specified("token_y", Some(validate_value(token_y.to_string())?))?; + Ok(builder) + } + + #[cfg(not(feature = "_sender-tcp"))] + { + let _ = token_y; + Err(error::fmt!( + ConfigError, + "cannot specify \"token_y\": ECDSA authentication is only available with ILP/TCP and not available with ILP/HTTP." + )) + } } /// Sets the ingestion protocol version. @@ -2270,7 +821,7 @@ impl SenderBuilder { Ok(self) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] /// Set the cumulative duration spent in retries. /// The value is in milliseconds, and the default is 10 seconds. pub fn retry_timeout(mut self, value: Duration) -> Result { @@ -2285,7 +836,7 @@ impl SenderBuilder { Ok(self) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] /// Set the minimum acceptable throughput while sending a buffer to the server. /// The sender will divide the payload size by this number to determine for how /// long to keep sending the payload before timing out. @@ -2308,7 +859,7 @@ impl SenderBuilder { Ok(self) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] /// Additional time to wait on top of that calculated from the minimum throughput. /// This accounts for the fixed latency of the HTTP request-response roundtrip. /// The default is 10 seconds. @@ -2332,7 +883,7 @@ impl SenderBuilder { Ok(self) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] /// Internal API, do not use. /// This is exposed exclusively for the Python client. /// We (QuestDB) use this to help us debug which client is being used if we encounter issues. @@ -2345,108 +896,28 @@ impl SenderBuilder { Ok(self) } - fn connect_tcp(&self, auth: &Option) -> Result { - let addr: SockAddr = gai::resolve_host_port(self.host.as_str(), self.port.as_str())?; - let mut sock = Socket::new(Domain::IPV4, Type::STREAM, Some(SockProtocol::TCP)) - .map_err(|io_err| map_io_to_socket_err("Could not open TCP socket: ", io_err))?; - - // See: https://idea.popcount.org/2014-04-03-bind-before-connect/ - // We set `SO_REUSEADDR` on the outbound socket to avoid issues where a client may exhaust - // their interface's ports. See: https://github.com/questdb/py-questdb-client/issues/21 - sock.set_reuse_address(true) - .map_err(|io_err| map_io_to_socket_err("Could not set SO_REUSEADDR: ", io_err))?; - - sock.set_linger(Some(Duration::from_secs(120))) - .map_err(|io_err| map_io_to_socket_err("Could not set socket linger: ", io_err))?; - sock.set_keepalive(true) - .map_err(|io_err| map_io_to_socket_err("Could not set SO_KEEPALIVE: ", io_err))?; - sock.set_nodelay(true) - .map_err(|io_err| map_io_to_socket_err("Could not set TCP_NODELAY: ", io_err))?; - if let Some(host) = self.net_interface.deref() { - let bind_addr = gai::resolve_host(host.as_str())?; - sock.bind(&bind_addr).map_err(|io_err| { - map_io_to_socket_err( - &format!("Could not bind to interface address {host:?}: "), - io_err, - ) - })?; - } - sock.connect(&addr).map_err(|io_err| { - let host_port = format!("{}:{}", self.host.deref(), *self.port); - let prefix = format!("Could not connect to {host_port:?}: "); - map_io_to_socket_err(&prefix, io_err) - })?; - - // We read during both TLS handshake and authentication. - // We set up a read timeout to prevent the client from "hanging" - // should we be connecting to a server configured in a different way - // from the client. - sock.set_read_timeout(Some(*self.auth_timeout)) - .map_err(|io_err| { - map_io_to_socket_err("Failed to set read timeout on socket: ", io_err) - })?; - - #[cfg(feature = "insecure-skip-verify")] - let tls_verify = *self.tls_verify; - - #[cfg(not(feature = "insecure-skip-verify"))] - let tls_verify = true; - - let mut conn = match configure_tls( - self.protocol.tls_enabled(), - tls_verify, - *self.tls_ca, - self.tls_roots.deref(), - )? { - Some(tls_config) => { - let server_name: ServerName = ServerName::try_from(self.host.as_str()) - .map_err(|inv_dns_err| error::fmt!(TlsError, "Bad host: {}", inv_dns_err))? - .to_owned(); - let mut tls_conn = - ClientConnection::new(tls_config, server_name).map_err(|rustls_err| { - error::fmt!(TlsError, "Could not create TLS client: {}", rustls_err) - })?; - while tls_conn.wants_write() || tls_conn.is_handshaking() { - tls_conn.complete_io(&mut sock).map_err(|io_err| { - if (io_err.kind() == ErrorKind::TimedOut) - || (io_err.kind() == ErrorKind::WouldBlock) - { - error::fmt!( - TlsError, - concat!( - "Failed to complete TLS handshake:", - " Timed out waiting for server ", - "response after {:?}." - ), - *self.auth_timeout - ) - } else { - error::fmt!(TlsError, "Failed to complete TLS handshake: {}", io_err) - } - })?; - } - Connection::Tls(StreamOwned::new(tls_conn, sock).into()) - } - None => Connection::Direct(sock), - }; - - if let Some(AuthParams::Ecdsa(auth)) = auth { - conn.authenticate(auth)?; - } - - Ok(ProtocolHandler::Socket(conn)) - } - - fn build_auth(&self) -> Result> { + fn build_auth(&self) -> Result> { match ( self.protocol, self.username.deref(), self.password.deref(), self.token.deref(), + + #[cfg(feature = "_sender-tcp")] self.token_x.deref(), + + #[cfg(not(feature = "_sender-tcp"))] + None::, + + #[cfg(feature = "_sender-tcp")] self.token_y.deref(), + + #[cfg(not(feature = "_sender-tcp"))] + None::, ) { (_, None, None, None, None, None) => Ok(None), + + #[cfg(feature = "_sender-tcp")] ( protocol, Some(username), @@ -2454,58 +925,64 @@ impl SenderBuilder { Some(token), Some(token_x), Some(token_y), - ) if protocol.is_tcpx() => Ok(Some(AuthParams::Ecdsa(EcdsaAuthParams { + ) if protocol.is_tcpx() => Ok(Some(conf::AuthParams::Ecdsa(conf::EcdsaAuthParams { key_id: username.to_string(), priv_key: token.to_string(), pub_key_x: token_x.to_string(), pub_key_y: token_y.to_string(), }))), + + #[cfg(feature = "_sender-tcp")] (protocol, Some(_username), Some(_password), None, None, None) if protocol.is_tcpx() => { Err(error::fmt!(ConfigError, r##"The "basic_auth" setting can only be used with the ILP/HTTP protocol."##, )) } + + #[cfg(feature = "_sender-tcp")] (protocol, None, None, Some(_token), None, None) if protocol.is_tcpx() => { Err(error::fmt!(ConfigError, "Token authentication only be used with the ILP/HTTP protocol.")) } + + #[cfg(feature = "_sender-tcp")] (protocol, _username, None, _token, _token_x, _token_y) if protocol.is_tcpx() => { Err(error::fmt!(ConfigError, r##"Incomplete ECDSA authentication parameters. Specify either all or none of: "username", "token", "token_x", "token_y"."##, )) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "_sender-http")] (protocol, Some(username), Some(password), None, None, None) if protocol.is_httpx() => { - Ok(Some(AuthParams::Basic(BasicAuthParams { + Ok(Some(conf::AuthParams::Basic(conf::BasicAuthParams { username: username.to_string(), password: password.to_string(), }))) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "_sender-http")] (protocol, Some(_username), None, None, None, None) if protocol.is_httpx() => { Err(error::fmt!(ConfigError, r##"Basic authentication parameter "username" is present, but "password" is missing."##, )) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "_sender-http")] (protocol, None, Some(_password), None, None, None) if protocol.is_httpx() => { Err(error::fmt!(ConfigError, r##"Basic authentication parameter "password" is present, but "username" is missing."##, )) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] (protocol, None, None, Some(token), None, None) if protocol.is_httpx() => { - Ok(Some(AuthParams::Token(TokenAuthParams { + Ok(Some(conf::AuthParams::Token(conf::TokenAuthParams { token: token.to_string(), }))) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] ( protocol, Some(_username), @@ -2516,7 +993,7 @@ impl SenderBuilder { ) if protocol.is_httpx() => { Err(error::fmt!(ConfigError, "ECDSA authentication is only available with ILP/TCP and not available with ILP/HTTP.")) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "_sender-http")] (protocol, _username, _password, _token, None, None) if protocol.is_httpx() => { Err(error::fmt!(ConfigError, @@ -2531,6 +1008,7 @@ impl SenderBuilder { } } + #[cfg(feature = "_sync-sender")] /// Build the sender. /// /// In the case of TCP, this synchronously establishes the TCP connection, and @@ -2546,11 +1024,30 @@ impl SenderBuilder { write!(descr, "tls=disabled,").unwrap(); } + #[cfg(feature = "insecure-skip-verify")] + let tls_verify = *self.tls_verify; + + let tls_settings = tls::TlsSettings::build( + self.protocol.tls_enabled(), + #[cfg(feature = "insecure-skip-verify")] + tls_verify, + *self.tls_ca, + self.tls_roots.deref().as_deref(), + )?; + let auth = self.build_auth()?; let handler = match self.protocol { - Protocol::Tcp | Protocol::Tcps => self.connect_tcp(&auth)?, - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-tcp")] + Protocol::Tcp | Protocol::Tcps => connect_tcp( + self.host.as_str(), + self.port.as_str(), + self.net_interface.deref().as_deref(), + *self.auth_timeout, + tls_settings, + &auth, + )?, + #[cfg(feature = "sync-sender-http")] Protocol::Http | Protocol::Https => { use ureq::unversioned::transport::Connector; use ureq::unversioned::transport::TcpConnector; @@ -2570,24 +1067,20 @@ impl SenderBuilder { .user_agent(user_agent) .no_delay(true); - #[cfg(feature = "insecure-skip-verify")] - let tls_verify = *self.tls_verify; - - #[cfg(not(feature = "insecure-skip-verify"))] - let tls_verify = true; + let tls_config = match tls_settings { + Some(tls_settings) => Some(tls::configure_tls(tls_settings)?), + None => None, + }; - let connector = connector.chain(TlsConnector::new(configure_tls( - self.protocol.tls_enabled(), - tls_verify, - *self.tls_ca, - self.tls_roots.deref(), - )?)); + let connector = connector.chain(TlsConnector::new(tls_config)); let auth = match auth { - Some(AuthParams::Basic(ref auth)) => Some(auth.to_header_string()), - Some(AuthParams::Token(ref auth)) => Some(auth.to_header_string()?), - Some(AuthParams::Ecdsa(_)) => { - return Err(error::fmt!( + Some(conf::AuthParams::Basic(ref auth)) => Some(auth.to_header_string()), + Some(conf::AuthParams::Token(ref auth)) => Some(auth.to_header_string()?), + + #[cfg(feature = "sync-sender-tcp")] + Some(conf::AuthParams::Ecdsa(_)) => { + return Err(fmt!( AuthError, "ECDSA authentication is not supported for ILP over HTTP. \ Please use basic or token authentication instead." @@ -2610,7 +1103,7 @@ impl SenderBuilder { self.host.deref(), self.port.deref() ); - ProtocolHandler::Http(HttpHandlerState { + SyncProtocolHandler::SyncHttp(SyncHttpHandlerState { agent, url, auth, @@ -2619,15 +1112,18 @@ impl SenderBuilder { } }; + #[allow(unused_mut)] let mut max_name_len = *self.max_name_len; let protocol_version = match self.protocol_version.deref() { Some(v) => *v, None => match self.protocol { + #[cfg(feature = "sync-sender-tcp")] Protocol::Tcp | Protocol::Tcps => ProtocolVersion::V1, - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] Protocol::Http | Protocol::Https => { - if let ProtocolHandler::Http(http_state) = &handler { + #[allow(irrefutable_let_patterns)] + if let SyncProtocolHandler::SyncHttp(http_state) = &handler { let settings_url = &format!( "{}://{}:{}/settings", self.protocol.schema(), @@ -2642,7 +1138,7 @@ impl SenderBuilder { } else if protocol_versions.contains(&ProtocolVersion::V1) { ProtocolVersion::V1 } else { - return Err(error::fmt!( + return Err(fmt!( ProtocolVersionError, "Server does not support current client" )); @@ -2660,23 +1156,23 @@ impl SenderBuilder { descr.push_str("auth=off]"); } - let sender = Sender { + let sender = Sender::new( descr, handler, - connected: true, - max_buf_size: *self.max_buf_size, + *self.max_buf_size, protocol_version, max_name_len, - }; + ); Ok(sender) } + #[cfg(feature = "_sender-tcp")] fn ensure_is_tcpx(&mut self, param_name: &str) -> Result<()> { if self.protocol.is_tcpx() { Ok(()) } else { - Err(error::fmt!( + Err(fmt!( ConfigError, "The {param_name:?} setting can only be used with the TCP protocol." )) @@ -2705,16 +1201,18 @@ where T::Err: std::fmt::Debug, { str_value.parse().map_err(|e| { - error::fmt!( + fmt!( ConfigError, "Could not parse {param_name:?} to number: {e:?}" ) }) } +#[cfg(feature = "_sender-tcp")] fn b64_decode(descr: &'static str, buf: &str) -> Result> { + use base64ct::{Base64UrlUnpadded, Encoding}; Base64UrlUnpadded::decode_vec(buf).map_err(|b64_err| { - error::fmt!( + fmt!( AuthError, "Misconfigured ILP authentication keys. Could not decode {}: {}. \ Hint: Check the keys for a possible typo.", @@ -2724,6 +1222,7 @@ fn b64_decode(descr: &'static str, buf: &str) -> Result> { }) } +#[cfg(feature = "_sender-tcp")] fn parse_public_key(pub_key_x: &str, pub_key_y: &str) -> Result> { let mut pub_key_x = b64_decode("public key x", pub_key_x)?; let mut pub_key_y = b64_decode("public key y", pub_key_y)?; @@ -2733,7 +1232,7 @@ fn parse_public_key(pub_key_x: &str, pub_key_y: &str) -> Result> { encoded.push(4u8); // 0x04 magic byte that identifies this as uncompressed. let pub_key_x_ken = pub_key_x.len(); if pub_key_x_ken > 32 { - return Err(error::fmt!( + return Err(fmt!( AuthError, "Misconfigured ILP authentication keys. Public key x is too long. \ Hint: Check the keys for a possible typo." @@ -2741,7 +1240,7 @@ fn parse_public_key(pub_key_x: &str, pub_key_y: &str) -> Result> { } let pub_key_y_len = pub_key_y.len(); if pub_key_y_len > 32 { - return Err(error::fmt!( + return Err(fmt!( AuthError, "Misconfigured ILP authentication keys. Public key y is too long. \ Hint: Check the keys for a possible typo." @@ -2754,7 +1253,8 @@ fn parse_public_key(pub_key_x: &str, pub_key_y: &str) -> Result> { Ok(encoded) } -fn parse_key_pair(auth: &EcdsaAuthParams) -> Result { +#[cfg(feature = "_sender-tcp")] +fn parse_key_pair(auth: &conf::EcdsaAuthParams) -> Result { let private_key = b64_decode("private authentication key", auth.priv_key.as_str())?; let public_key = parse_public_key(auth.pub_key_x.as_str(), auth.pub_key_y.as_str())?; @@ -2777,7 +1277,7 @@ fn parse_key_pair(auth: &EcdsaAuthParams) -> Result { }; res.map_err(|key_rejected| { - error::fmt!( + fmt!( AuthError, "Misconfigured ILP authentication keys: {}. Hint: Check the keys for a possible typo.", key_rejected @@ -2785,275 +1285,33 @@ fn parse_key_pair(auth: &EcdsaAuthParams) -> Result { }) } -pub(crate) struct F64Serializer { - buf: ryu::Buffer, - n: f64, -} - -impl F64Serializer { - pub(crate) fn new(n: f64) -> Self { - F64Serializer { - buf: ryu::Buffer::new(), - n, - } - } - - // This function was taken and customized from the ryu crate. - #[cold] - fn format_nonfinite(&self) -> &'static str { - const MANTISSA_MASK: u64 = 0x000fffffffffffff; - const SIGN_MASK: u64 = 0x8000000000000000; - let bits = self.n.to_bits(); - if bits & MANTISSA_MASK != 0 { - "NaN" - } else if bits & SIGN_MASK != 0 { - "-Infinity" - } else { - "Infinity" - } - } - - pub(crate) fn as_str(&mut self) -> &str { - if self.n.is_finite() { - self.buf.format_finite(self.n) - } else { - self.format_nonfinite() - } - } -} - -impl Sender { - /// Create a new `Sender` instance from the given configuration string. - /// - /// The format of the string is: `"http::addr=host:port;key=value;...;"`. - /// - /// Instead of `"http"`, you can also specify `"https"`, `"tcp"`, and `"tcps"`. - /// - /// We recommend HTTP for most cases because it provides more features, like - /// reporting errors to the client and supporting transaction control. TCP can - /// sometimes be faster in higher-latency networks, but misses a number of - /// features. - /// - /// Keys in the config string correspond to same-named methods on `SenderBuilder`. - /// - /// For the full list of keys and values, see the docs on [`SenderBuilder`]. - /// - /// You can also load the configuration from an environment variable. - /// See [`Sender::from_env`]. - /// - /// In the case of TCP, this synchronously establishes the TCP connection, and - /// returns once the connection is fully established. If the connection - /// requires authentication or TLS, these will also be completed before - /// returning. - pub fn from_conf>(conf: T) -> Result { - SenderBuilder::from_conf(conf)?.build() - } - - /// Create a new `Sender` from the configuration stored in the `QDB_CLIENT_CONF` - /// environment variable. The format is the same as that accepted by - /// [`Sender::from_conf`]. - /// - /// In the case of TCP, this synchronously establishes the TCP connection, and - /// returns once the connection is fully established. If the connection - /// requires authentication or TLS, these will also be completed before - /// returning. - pub fn from_env() -> Result { - SenderBuilder::from_env()?.build() - } - - /// Creates a new [`Buffer`] using the sender's protocol settings - pub fn new_buffer(&self) -> Buffer { - Buffer::with_max_name_len(self.protocol_version, self.max_name_len) - } - - #[allow(unused_variables)] - fn flush_impl(&mut self, buf: &Buffer, transactional: bool) -> Result<()> { - if !self.connected { - return Err(error::fmt!( - SocketError, - "Could not flush buffer: not connected to database." - )); - } - buf.check_can_flush()?; - - if buf.len() > self.max_buf_size { - return Err(error::fmt!( - InvalidApiCall, - "Could not flush buffer: Buffer size of {} exceeds maximum configured allowed size of {} bytes.", - buf.len(), - self.max_buf_size - )); - } +struct DebugBytes<'a>(pub &'a [u8]); - self.check_protocol_version(buf.version)?; +impl<'a> Debug for DebugBytes<'a> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "b\"")?; - let bytes = buf.as_bytes(); - if bytes.is_empty() { - return Ok(()); - } - match self.handler { - ProtocolHandler::Socket(ref mut conn) => { - if transactional { - return Err(error::fmt!( - InvalidApiCall, - "Transactional flushes are not supported for ILP over TCP." - )); - } - conn.write_all(bytes).map_err(|io_err| { - self.connected = false; - map_io_to_socket_err("Could not flush buffer: ", io_err) - })?; - } - #[cfg(feature = "ilp-over-http")] - ProtocolHandler::Http(ref state) => { - if transactional && !buf.transactional() { - return Err(error::fmt!( - InvalidApiCall, - "Buffer contains lines for multiple tables. \ - Transactional flushes are only supported for buffers containing lines for a single table." - )); + for &byte in self.0 { + match byte { + // Printable ASCII characters (except backslash and quote) + 0x20..=0x21 | 0x23..=0x5B | 0x5D..=0x7E => { + write!(f, "{}", byte as char)?; } - let request_min_throughput = *state.config.request_min_throughput; - let extra_time = if request_min_throughput > 0 { - (bytes.len() as f64) / (request_min_throughput as f64) - } else { - 0.0f64 - }; - - return match http_send_with_retries( - state, - bytes, - *state.config.request_timeout + Duration::from_secs_f64(extra_time), - *state.config.retry_timeout, - ) { - Ok(res) => { - if res.status().is_client_error() || res.status().is_server_error() { - Err(parse_http_error(res.status().as_u16(), res)) - } else { - res.into_body(); - Ok(()) - } - } - Err(err) => Err(Error::from_ureq_error(err, &state.url)), - }; + // Common escape sequences + b'\n' => write!(f, "\\n")?, + b'\r' => write!(f, "\\r")?, + b'\t' => write!(f, "\\t")?, + b'\\' => write!(f, "\\\\")?, + b'"' => write!(f, "\\\"")?, + b'\0' => write!(f, "\\0")?, + // Non-printable bytes as hex escapes + _ => write!(f, "\\x{byte:02x}")?, } } - Ok(()) - } - - /// Send the batch of rows in the buffer to the QuestDB server, and, if the - /// `transactional` parameter is true, ensure the flush will be transactional. - /// - /// A flush is transactional iff all the rows belong to the same table. This allows - /// QuestDB to treat the flush as a single database transaction, because it doesn't - /// support transactions spanning multiple tables. Additionally, only ILP-over-HTTP - /// supports transactional flushes. - /// - /// If the flush wouldn't be transactional, this function returns an error and - /// doesn't flush any data. - /// - /// The function sends an HTTP request and waits for the response. If the server - /// responds with an error, it returns a descriptive error. In the case of a network - /// error, it retries until it has exhausted the retry time budget. - /// - /// All the data stays in the buffer. Clear the buffer before starting a new batch. - #[cfg(feature = "ilp-over-http")] - pub fn flush_and_keep_with_flags(&mut self, buf: &Buffer, transactional: bool) -> Result<()> { - self.flush_impl(buf, transactional) - } - - /// Send the given buffer of rows to the QuestDB server. - /// - /// All the data stays in the buffer. Clear the buffer before starting a new batch. - /// - /// To send and clear in one step, call [Sender::flush] instead. - pub fn flush_and_keep(&mut self, buf: &Buffer) -> Result<()> { - self.flush_impl(buf, false) - } - - /// Send the given buffer of rows to the QuestDB server, clearing the buffer. - /// - /// After this function returns, the buffer is empty and ready for the next batch. - /// If you want to preserve the buffer contents, call [Sender::flush_and_keep]. If - /// you want to ensure the flush is transactional, call - /// [Sender::flush_and_keep_with_flags]. - /// - /// With ILP-over-HTTP, this function sends an HTTP request and waits for the - /// response. If the server responds with an error, it returns a descriptive error. - /// In the case of a network error, it retries until it has exhausted the retry time - /// budget. - /// - /// With ILP-over-TCP, the function blocks only until the buffer is flushed to the - /// underlying OS-level network socket, without waiting to actually send it to the - /// server. In the case of an error, the server will quietly disconnect: consult the - /// server logs for error messages. - /// - /// HTTP should be the first choice, but use TCP if you need to continuously send - /// data to the server at a high rate. - /// - /// To improve the HTTP performance, send larger buffers (with more rows), and - /// consider parallelizing writes using multiple senders from multiple threads. - pub fn flush(&mut self, buf: &mut Buffer) -> Result<()> { - self.flush_impl(buf, false)?; - buf.clear(); - Ok(()) - } - - /// Tell whether the sender is no longer usable and must be dropped. - /// - /// This happens when there was an earlier failure. - /// - /// This method is specific to ILP-over-TCP and is not relevant for ILP-over-HTTP. - pub fn must_close(&self) -> bool { - !self.connected - } - - /// Returns the sender's protocol version - /// - /// - Explicitly set version, or - /// - Auto-detected for HTTP transport, or [`ProtocolVersion::V1`] for TCP transport. - pub fn protocol_version(&self) -> ProtocolVersion { - self.protocol_version - } - - /// Return the sender's maxinum name length of any column or table name. - /// This is either set explicitly when constructing the sender, - /// or the default value of 127. - /// When unset and using protocol version 2 over HTTP, the value is read - /// from the server from the `cairo.max.file.name.length` setting in - /// `server.conf` which defaults to 127. - pub fn max_name_len(&self) -> usize { - self.max_name_len - } - #[inline(always)] - fn check_protocol_version(&self, version: ProtocolVersion) -> Result<()> { - if self.protocol_version != version { - return Err(error::fmt!( - ProtocolVersionError, - "Attempting to send with protocol version {} \ - but the sender is configured to use protocol version {}", - version, - self.protocol_version - )); - } - Ok(()) + write!(f, "\"") } } -pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; -pub(crate) const DOUBLE_BINARY_FORMAT_TYPE: u8 = 16; - -mod conf; -pub(crate) mod ndarr; -mod timestamp; - -#[cfg(feature = "ilp-over-http")] -mod http; - -use crate::ingress::ndarr::check_and_get_array_bytes_size; -#[cfg(feature = "ilp-over-http")] -use http::*; - #[cfg(test)] mod tests; diff --git a/questdb-rs/src/ingress/http.rs b/questdb-rs/src/ingress/sender/http.rs similarity index 89% rename from questdb-rs/src/ingress/http.rs rename to questdb-rs/src/ingress/sender/http.rs index c4b81996..3a99be8e 100644 --- a/questdb-rs/src/ingress/http.rs +++ b/questdb-rs/src/ingress/sender/http.rs @@ -22,11 +22,8 @@ * ******************************************************************************/ -use super::conf::ConfigSetting; use crate::error::fmt; use crate::{error, Error}; -use base64ct::Base64; -use base64ct::Encoding; use rand::Rng; use rustls::{ClientConnection, StreamOwned}; use rustls_pki_types::ServerName; @@ -41,75 +38,28 @@ use ureq::unversioned::transport::{ Buffers, Connector, LazyBuffers, NextTimeout, Transport, TransportAdapter, }; +use crate::ingress::conf::HttpConfig; use crate::ingress::ProtocolVersion; use ureq::unversioned::*; use ureq::{http, Body}; -#[derive(PartialEq, Debug, Clone)] -pub(super) struct BasicAuthParams { - pub(super) username: String, - pub(super) password: String, -} - -impl BasicAuthParams { - pub(super) fn to_header_string(&self) -> String { - let pair = format!("{}:{}", self.username, self.password); - let encoded = Base64::encode_string(pair.as_bytes()); - format!("Basic {encoded}") - } -} - -#[derive(PartialEq, Debug, Clone)] -pub(super) struct TokenAuthParams { - pub(super) token: String, -} - -impl TokenAuthParams { - pub(super) fn to_header_string(&self) -> crate::Result { - if self.token.contains('\n') { - return Err(error::fmt!( - AuthError, - "Bad auth token: Should not contain new-line char." - )); - } - Ok(format!("Bearer {}", self.token)) - } -} - -#[derive(Debug, Clone)] -pub(super) struct HttpConfig { - pub(super) request_min_throughput: ConfigSetting, - pub(super) user_agent: String, - pub(super) retry_timeout: ConfigSetting, - pub(super) request_timeout: ConfigSetting, -} - -impl Default for HttpConfig { - fn default() -> Self { - Self { - request_min_throughput: ConfigSetting::new_default(102400), // 100 KiB/s - user_agent: concat!("questdb/rust/", env!("CARGO_PKG_VERSION")).to_string(), - retry_timeout: ConfigSetting::new_default(Duration::from_secs(10)), - request_timeout: ConfigSetting::new_default(Duration::from_secs(10)), - } - } -} - -pub(super) struct HttpHandlerState { +#[cfg(feature = "sync-sender-http")] +pub(crate) struct SyncHttpHandlerState { /// Maintains a pool of open HTTP connections to the endpoint. - pub(super) agent: ureq::Agent, + pub(crate) agent: ureq::Agent, /// The URL of the HTTP endpoint. - pub(super) url: String, + pub(crate) url: String, /// The content of the `Authorization` HTTP header. - pub(super) auth: Option, + pub(crate) auth: Option, /// HTTP params configured via the `SenderBuilder`. - pub(super) config: HttpConfig, + pub(crate) config: HttpConfig, } -impl HttpHandlerState { +#[cfg(feature = "sync-sender-http")] +impl SyncHttpHandlerState { fn send_request( &self, buf: &[u8], @@ -155,7 +105,7 @@ impl HttpHandlerState { } #[derive(Debug)] -pub struct TlsConnector { +pub(crate) struct TlsConnector { tls_config: Option>, } @@ -166,7 +116,7 @@ impl Connector for TlsConnector { &self, details: &transport::ConnectionDetails, chained: Option, - ) -> std::result::Result, ureq::Error> { + ) -> Result, ureq::Error> { let transport = match chained { Some(t) => t, None => return Ok(None), @@ -384,7 +334,7 @@ pub(super) fn parse_http_error(http_status_code: u16, response: Response) #[allow(clippy::result_large_err)] // `ureq::Error` is large enough to cause this warning. fn retry_http_send( - state: &HttpHandlerState, + state: &SyncHttpHandlerState, buf: &[u8], request_timeout: Duration, retry_timeout: Duration, @@ -417,7 +367,7 @@ fn retry_http_send( #[allow(clippy::result_large_err)] // `ureq::Error` is large enough to cause this warning. pub(super) fn http_send_with_retries( - state: &HttpHandlerState, + state: &SyncHttpHandlerState, buf: &[u8], request_timeout: Duration, retry_timeout: Duration, @@ -437,8 +387,8 @@ pub(super) fn http_send_with_retries( /// /// If the server does not support the `/settings` endpoint (404), it returns /// default values. -pub(super) fn read_server_settings( - state: &HttpHandlerState, +pub(crate) fn read_server_settings( + state: &SyncHttpHandlerState, settings_url: &str, default_max_name_len: usize, ) -> Result<(Vec, usize), Error> { @@ -540,7 +490,7 @@ pub(super) fn read_server_settings( #[allow(clippy::result_large_err)] // `ureq::Error` is large enough to cause this warning. fn retry_http_get( - state: &HttpHandlerState, + state: &SyncHttpHandlerState, url: &str, request_timeout: Duration, retry_timeout: Duration, @@ -573,7 +523,7 @@ fn retry_http_get( #[allow(clippy::result_large_err)] // `ureq::Error` is large enough to cause this warning. fn http_get_with_retries( - state: &HttpHandlerState, + state: &SyncHttpHandlerState, url: &str, request_timeout: Duration, retry_timeout: Duration, diff --git a/questdb-rs/src/ingress/sender/mod.rs b/questdb-rs/src/ingress/sender/mod.rs new file mode 100644 index 00000000..6b65cce2 --- /dev/null +++ b/questdb-rs/src/ingress/sender/mod.rs @@ -0,0 +1,314 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +use crate::error::{self, Result}; +use crate::ingress::{Buffer, ProtocolVersion, SenderBuilder}; +use std::fmt::{Debug, Formatter}; + +#[cfg(feature = "sync-sender-tcp")] +mod tcp; + +#[cfg(feature = "sync-sender-tcp")] +pub(crate) use tcp::*; + +#[cfg(feature = "sync-sender-tcp")] +use std::io::Write; + +#[cfg(feature = "sync-sender-tcp")] +use crate::ingress::map_io_to_socket_err; + +#[cfg(feature = "sync-sender-http")] +mod http; + +#[cfg(feature = "sync-sender-http")] +pub(crate) use http::*; + +pub(crate) enum SyncProtocolHandler { + #[cfg(feature = "sync-sender-tcp")] + SyncTcp(SyncConnection), + + #[cfg(feature = "sync-sender-http")] + SyncHttp(SyncHttpHandlerState), +} + +/// Connects to a QuestDB instance and inserts data via the ILP protocol. +/// +/// * To construct an instance, use [`Sender::from_conf`] or the [`SenderBuilder`]. +/// * To prepare messages, use [`Buffer`] objects. +/// * To send messages, call the [`flush`](Sender::flush) method. +pub struct Sender { + descr: String, + handler: SyncProtocolHandler, + connected: bool, + max_buf_size: usize, + protocol_version: ProtocolVersion, + max_name_len: usize, +} + +impl Debug for Sender { + fn fmt(&self, f: &mut Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { + f.write_str(self.descr.as_str()) + } +} + +impl Sender { + pub(crate) fn new( + descr: String, + handler: SyncProtocolHandler, + max_buf_size: usize, + protocol_version: ProtocolVersion, + max_name_len: usize, + ) -> Self { + Self { + descr, + handler, + connected: true, + max_buf_size, + protocol_version, + max_name_len, + } + } + + /// Create a new `Sender` instance from the given configuration string. + /// + /// The format of the string is: `"http::addr=host:port;key=value;...;"`. + /// + /// Instead of `"http"`, you can also specify `"https"`, `"tcp"`, and `"tcps"`. + /// + /// We recommend HTTP for most cases because it provides more features, like + /// reporting errors to the client and supporting transaction control. TCP can + /// sometimes be faster in higher-latency networks, but misses a number of + /// features. + /// + /// Keys in the config string correspond to same-named methods on `SenderBuilder`. + /// + /// For the full list of keys and values, see the docs on [`SenderBuilder`]. + /// + /// You can also load the configuration from an environment variable. + /// See [`Sender::from_env`]. + /// + /// In the case of TCP, this synchronously establishes the TCP connection, and + /// returns once the connection is fully established. If the connection + /// requires authentication or TLS, these will also be completed before + /// returning. + pub fn from_conf>(conf: T) -> Result { + SenderBuilder::from_conf(conf)?.build() + } + + /// Create a new `Sender` from the configuration stored in the `QDB_CLIENT_CONF` + /// environment variable. The format is the same as that accepted by + /// [`Sender::from_conf`]. + /// + /// In the case of TCP, this synchronously establishes the TCP connection, and + /// returns once the connection is fully established. If the connection + /// requires authentication or TLS, these will also be completed before + /// returning. + pub fn from_env() -> Result { + SenderBuilder::from_env()?.build() + } + + /// Creates a new [`Buffer`] using the sender's protocol settings + pub fn new_buffer(&self) -> Buffer { + Buffer::with_max_name_len(self.protocol_version, self.max_name_len) + } + + #[allow(unused_variables)] + fn flush_impl(&mut self, buf: &Buffer, transactional: bool) -> Result<()> { + if !self.connected { + return Err(error::fmt!( + SocketError, + "Could not flush buffer: not connected to database." + )); + } + buf.check_can_flush()?; + + if buf.len() > self.max_buf_size { + return Err(error::fmt!( + InvalidApiCall, + "Could not flush buffer: Buffer size of {} exceeds maximum configured allowed size of {} bytes.", + buf.len(), + self.max_buf_size + )); + } + + self.check_protocol_version(buf.protocol_version())?; + + let bytes = buf.as_bytes(); + if bytes.is_empty() { + return Ok(()); + } + match self.handler { + #[cfg(feature = "sync-sender-tcp")] + SyncProtocolHandler::SyncTcp(ref mut conn) => { + if transactional { + return Err(error::fmt!( + InvalidApiCall, + "Transactional flushes are not supported for ILP over TCP." + )); + } + conn.write_all(bytes).map_err(|io_err| { + self.connected = false; + map_io_to_socket_err("Could not flush buffer: ", io_err) + })?; + conn.flush().map_err(|io_err| { + self.connected = false; + map_io_to_socket_err("Could not flush to network: ", io_err) + })?; + Ok(()) + } + #[cfg(feature = "sync-sender-http")] + SyncProtocolHandler::SyncHttp(ref state) => { + if transactional && !buf.transactional() { + return Err(error::fmt!( + InvalidApiCall, + "Buffer contains lines for multiple tables. \ + Transactional flushes are only supported for buffers containing lines for a single table." + )); + } + let request_min_throughput = *state.config.request_min_throughput; + let extra_time = if request_min_throughput > 0 { + (bytes.len() as f64) / (request_min_throughput as f64) + } else { + 0.0f64 + }; + + match http_send_with_retries( + state, + bytes, + *state.config.request_timeout + std::time::Duration::from_secs_f64(extra_time), + *state.config.retry_timeout, + ) { + Ok(res) => { + if res.status().is_client_error() || res.status().is_server_error() { + Err(parse_http_error(res.status().as_u16(), res)) + } else { + res.into_body(); + Ok(()) + } + } + Err(err) => Err(crate::error::Error::from_ureq_error(err, &state.url)), + } + } + } + } + + /// Send the batch of rows in the buffer to the QuestDB server, and, if the + /// `transactional` parameter is true, ensure the flush will be transactional. + /// + /// A flush is transactional iff all the rows belong to the same table. This allows + /// QuestDB to treat the flush as a single database transaction, because it doesn't + /// support transactions spanning multiple tables. Additionally, only ILP-over-HTTP + /// supports transactional flushes. + /// + /// If the flush wouldn't be transactional, this function returns an error and + /// doesn't flush any data. + /// + /// The function sends an HTTP request and waits for the response. If the server + /// responds with an error, it returns a descriptive error. In the case of a network + /// error, it retries until it has exhausted the retry time budget. + /// + /// All the data stays in the buffer. Clear the buffer before starting a new batch. + #[cfg(feature = "sync-sender-http")] + pub fn flush_and_keep_with_flags(&mut self, buf: &Buffer, transactional: bool) -> Result<()> { + self.flush_impl(buf, transactional) + } + + /// Send the given buffer of rows to the QuestDB server. + /// + /// All the data stays in the buffer. Clear the buffer before starting a new batch. + /// + /// To send and clear in one step, call [Sender::flush] instead. + pub fn flush_and_keep(&mut self, buf: &Buffer) -> Result<()> { + self.flush_impl(buf, false) + } + + /// Send the given buffer of rows to the QuestDB server, clearing the buffer. + /// + /// After this function returns, the buffer is empty and ready for the next batch. + /// If you want to preserve the buffer contents, call [Sender::flush_and_keep]. If + /// you want to ensure the flush is transactional, call + /// [Sender::flush_and_keep_with_flags]. + /// + /// With ILP-over-HTTP, this function sends an HTTP request and waits for the + /// response. If the server responds with an error, it returns a descriptive error. + /// In the case of a network error, it retries until it has exhausted the retry time + /// budget. + /// + /// With ILP-over-TCP, the function blocks only until the buffer is flushed to the + /// underlying OS-level network socket, without waiting to actually send it to the + /// server. In the case of an error, the server will quietly disconnect: consult the + /// server logs for error messages. + /// + /// HTTP should be the first choice, but use TCP if you need to continuously send + /// data to the server at a high rate. + /// + /// To improve the HTTP performance, send larger buffers (with more rows), and + /// consider parallelizing writes using multiple senders from multiple threads. + pub fn flush(&mut self, buf: &mut Buffer) -> crate::Result<()> { + self.flush_impl(buf, false)?; + buf.clear(); + Ok(()) + } + + /// Tell whether the sender is no longer usable and must be dropped. + /// + /// This happens when there was an earlier failure. + /// + /// This method is specific to ILP-over-TCP and is not relevant for ILP-over-HTTP. + pub fn must_close(&self) -> bool { + !self.connected + } + + /// Returns the sender's protocol version + /// + /// - Explicitly set version, or + /// - Auto-detected for HTTP transport, or [`ProtocolVersion::V1`] for TCP transport. + pub fn protocol_version(&self) -> ProtocolVersion { + self.protocol_version + } + + /// Return the sender's maxinum name length of any column or table name. + /// This is either set explicitly when constructing the sender, + /// or the default value of 127. + /// When unset and using protocol version 2 over HTTP, the value is read + /// from the server from the `cairo.max.file.name.length` setting in + /// `server.conf` which defaults to 127. + pub fn max_name_len(&self) -> usize { + self.max_name_len + } + + #[inline(always)] + fn check_protocol_version(&self, version: ProtocolVersion) -> Result<()> { + if self.protocol_version != version { + return Err(error::fmt!( + ProtocolVersionError, + "Attempting to send with protocol version {} \ + but the sender is configured to use protocol version {}", + version, + self.protocol_version + )); + } + Ok(()) + } +} diff --git a/questdb-rs/src/ingress/sender/tcp.rs b/questdb-rs/src/ingress/sender/tcp.rs new file mode 100644 index 00000000..e9a2e91e --- /dev/null +++ b/questdb-rs/src/ingress/sender/tcp.rs @@ -0,0 +1,245 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +use crate::error; +use crate::gai; +use crate::ingress::tls::{configure_tls, TlsSettings}; +use crate::ingress::{conf, map_io_to_socket_err, parse_key_pair, SyncProtocolHandler}; +use rustls::{ClientConnection, StreamOwned}; +use rustls_pki_types::ServerName; +use socket2::{Domain, Protocol as SockProtocol, SockAddr, Socket, Type}; +use std::io::{self, BufReader, Write as IoWrite}; +use std::io::{BufRead, ErrorKind}; +use std::time::Duration; + +#[cfg(feature = "aws-lc-crypto")] +use aws_lc_rs::rand::SystemRandom; + +#[cfg(feature = "ring-crypto")] +use ring::rand::SystemRandom; + +pub(crate) enum SyncConnection { + Direct(Socket), + Tls(Box>), +} + +impl SyncConnection { + fn send_key_id(&mut self, key_id: &str) -> crate::Result<()> { + writeln!(self, "{key_id}") + .map_err(|io_err| map_io_to_socket_err("Failed to send key_id: ", io_err))?; + Ok(()) + } + + fn read_challenge(&mut self) -> crate::Result> { + let mut buf = Vec::new(); + let mut reader = BufReader::new(self); + reader.read_until(b'\n', &mut buf).map_err(|io_err| { + map_io_to_socket_err( + "Failed to read authentication challenge (timed out?): ", + io_err, + ) + })?; + if buf.last().copied().unwrap_or(b'\0') != b'\n' { + return Err(if buf.is_empty() { + error::fmt!( + AuthError, + concat!( + "Did not receive auth challenge. ", + "Is the database configured to require ", + "authentication?" + ) + ) + } else { + error::fmt!(AuthError, "Received incomplete auth challenge: {:?}", buf) + }); + } + buf.pop(); // b'\n' + Ok(buf) + } + + pub(crate) fn authenticate( + &mut self, + auth: &crate::ingress::conf::EcdsaAuthParams, + ) -> crate::Result<()> { + use base64ct::{Base64, Encoding}; + + if auth.key_id.contains('\n') { + return Err(error::fmt!( + AuthError, + "Bad key id {:?}: Should not contain new-line char.", + auth.key_id + )); + } + let key_pair = parse_key_pair(auth)?; + self.send_key_id(auth.key_id.as_str())?; + let challenge = self.read_challenge()?; + let rng = SystemRandom::new(); + let signature = key_pair + .sign(&rng, &challenge[..]) + .map_err(|unspecified_err| { + error::fmt!(AuthError, "Failed to sign challenge: {}", unspecified_err) + })?; + let mut encoded_sig = Base64::encode_string(signature.as_ref()); + encoded_sig.push('\n'); + let buf = encoded_sig.as_bytes(); + if let Err(io_err) = self.write_all(buf) { + return Err(map_io_to_socket_err( + "Could not send signed challenge: ", + io_err, + )); + } + Ok(()) + } +} + +impl io::Read for SyncConnection { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + Self::Direct(sock) => sock.read(buf), + Self::Tls(stream) => stream.read(buf), + } + } +} + +impl IoWrite for SyncConnection { + fn write(&mut self, buf: &[u8]) -> io::Result { + match self { + Self::Direct(sock) => sock.write(buf), + Self::Tls(stream) => stream.write(buf), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self { + Self::Direct(sock) => sock.flush(), + Self::Tls(stream) => stream.flush(), + } + } +} + +// This is important to make sure that any Windows socket is properly closed +// without dropping in-flight writes. +// We also set SO_LINGER to 120, but that is not enough apparently. +impl Drop for SyncProtocolHandler { + fn drop(&mut self) { + if let SyncProtocolHandler::SyncTcp(conn) = self { + match conn { + SyncConnection::Direct(sock) => { + let _ = sock.shutdown(std::net::Shutdown::Write); + } + SyncConnection::Tls(stream) => { + let _ = stream.get_ref().shutdown(std::net::Shutdown::Write); + } + } + } + } +} + +pub(crate) fn connect_tcp( + host: &str, + port: &str, + net_interface: Option<&str>, + auth_timeout: Duration, + tls_settings: Option, + auth: &Option, +) -> crate::Result { + let addr: SockAddr = gai::resolve_host_port(host, port)?; + let mut sock = Socket::new(Domain::IPV4, Type::STREAM, Some(SockProtocol::TCP)) + .map_err(|io_err| map_io_to_socket_err("Could not open TCP socket: ", io_err))?; + + // See: https://idea.popcount.org/2014-04-03-bind-before-connect/ + // We set `SO_REUSEADDR` on the outbound socket to avoid issues where a client may exhaust + // their interface's ports. See: https://github.com/questdb/py-questdb-client/issues/21 + sock.set_reuse_address(true) + .map_err(|io_err| map_io_to_socket_err("Could not set SO_REUSEADDR: ", io_err))?; + + sock.set_linger(Some(Duration::from_secs(120))) + .map_err(|io_err| map_io_to_socket_err("Could not set socket linger: ", io_err))?; + sock.set_keepalive(true) + .map_err(|io_err| map_io_to_socket_err("Could not set SO_KEEPALIVE: ", io_err))?; + sock.set_nodelay(true) + .map_err(|io_err| map_io_to_socket_err("Could not set TCP_NODELAY: ", io_err))?; + if let Some(host) = net_interface { + let bind_addr = gai::resolve_host(host)?; + sock.bind(&bind_addr).map_err(|io_err| { + map_io_to_socket_err( + &format!("Could not bind to interface address {host:?}: "), + io_err, + ) + })?; + } + + sock.connect(&addr).map_err(|io_err| { + let host_port = format!("{host}:{port}"); + let prefix = format!("Could not connect to {host_port:?}: "); + map_io_to_socket_err(&prefix, io_err) + })?; + + // We read during both TLS handshake and authentication. + // We set up a read timeout to prevent the client from "hanging" + // should we be connecting to a server configured in a different way + // from the client. + sock.set_read_timeout(Some(auth_timeout)) + .map_err(|io_err| map_io_to_socket_err("Failed to set read timeout on socket: ", io_err))?; + + let mut conn = match tls_settings { + Some(tls_settings) => { + let tls_config = configure_tls(tls_settings)?; + let server_name: ServerName = ServerName::try_from(host) + .map_err(|inv_dns_err| error::fmt!(TlsError, "Bad host: {}", inv_dns_err))? + .to_owned(); + let mut tls_conn = + ClientConnection::new(tls_config, server_name).map_err(|rustls_err| { + error::fmt!(TlsError, "Could not create TLS client: {}", rustls_err) + })?; + while tls_conn.wants_write() || tls_conn.is_handshaking() { + tls_conn.complete_io(&mut sock).map_err(|io_err| { + if (io_err.kind() == ErrorKind::TimedOut) + || (io_err.kind() == ErrorKind::WouldBlock) + { + error::fmt!( + TlsError, + concat!( + "Failed to complete TLS handshake:", + " Timed out waiting for server ", + "response after {:?}." + ), + auth_timeout + ) + } else { + error::fmt!(TlsError, "Failed to complete TLS handshake: {}", io_err) + } + })?; + } + SyncConnection::Tls(StreamOwned::new(tls_conn, sock).into()) + } + None => SyncConnection::Direct(sock), + }; + + if let Some(conf::AuthParams::Ecdsa(auth)) = auth { + conn.authenticate(auth)?; + } + + Ok(SyncProtocolHandler::SyncTcp(conn)) +} diff --git a/questdb-rs/src/ingress/tests.rs b/questdb-rs/src/ingress/tests.rs index 0d896e2d..d4e40a8f 100644 --- a/questdb-rs/src/ingress/tests.rs +++ b/questdb-rs/src/ingress/tests.rs @@ -24,9 +24,11 @@ use super::*; use crate::ErrorCode; + +#[cfg(feature = "sync-sender-tcp")] use tempfile::TempDir; -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] #[test] fn http_simple() { let builder = SenderBuilder::from_conf("http::addr=127.0.0.1;").unwrap(); @@ -36,7 +38,7 @@ fn http_simple() { assert!(!builder.protocol.tls_enabled()); } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] #[test] fn https_simple() { let builder = SenderBuilder::from_conf("https::addr=localhost;").unwrap(); @@ -52,6 +54,7 @@ fn https_simple() { assert_defaulted_eq(&builder.tls_ca, CertificateAuthority::OsRoots); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn tcp_simple() { let builder = SenderBuilder::from_conf("tcp::addr=127.0.0.1;").unwrap(); @@ -61,6 +64,7 @@ fn tcp_simple() { assert!(!builder.protocol.tls_enabled()); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn tcps_simple() { let builder = SenderBuilder::from_conf("tcps::addr=localhost;").unwrap(); @@ -76,6 +80,7 @@ fn tcps_simple() { assert_defaulted_eq(&builder.tls_ca, CertificateAuthority::OsRoots); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn invalid_value() { assert_conf_err( @@ -84,6 +89,7 @@ fn invalid_value() { ); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn specified_cant_change() { let mut builder = SenderBuilder::from_conf("tcp::addr=localhost;").unwrap(); @@ -94,6 +100,7 @@ fn specified_cant_change() { ); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn missing_addr() { assert_conf_err( @@ -102,6 +109,7 @@ fn missing_addr() { ); } +#[cfg(any(feature = "sync-sender-tcp", feature = "sync-sender-http"))] #[test] fn unsupported_service() { assert_conf_err( @@ -110,7 +118,7 @@ fn unsupported_service() { ); } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] #[test] fn http_basic_auth() { let builder = @@ -118,7 +126,7 @@ fn http_basic_auth() { .unwrap(); let auth = builder.build_auth().unwrap(); match auth.unwrap() { - AuthParams::Basic(BasicAuthParams { username, password }) => { + conf::AuthParams::Basic(conf::BasicAuthParams { username, password }) => { assert_eq!(username, "user123"); assert_eq!(password, "pass321"); } @@ -128,13 +136,13 @@ fn http_basic_auth() { } } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] #[test] fn http_token_auth() { let builder = SenderBuilder::from_conf("http::addr=localhost:9000;token=token123;").unwrap(); let auth = builder.build_auth().unwrap(); match auth.unwrap() { - AuthParams::Token(TokenAuthParams { token }) => { + conf::AuthParams::Token(conf::TokenAuthParams { token }) => { assert_eq!(token, "token123"); } _ => { @@ -143,7 +151,7 @@ fn http_token_auth() { } } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] #[test] fn incomplete_basic_auth() { assert_conf_err( @@ -160,7 +168,7 @@ fn incomplete_basic_auth() { ); } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] #[test] fn zero_timeout_forbidden() { assert_conf_err( @@ -175,7 +183,7 @@ fn zero_timeout_forbidden() { ); } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] #[test] fn misspelled_basic_auth() { assert_conf_err( @@ -188,7 +196,7 @@ fn misspelled_basic_auth() { ); } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] #[test] fn inconsistent_http_auth() { let expected_err_msg = r##"Inconsistent HTTP authentication parameters. Specify either "username" and "password", or just "token"."##; @@ -202,7 +210,7 @@ fn inconsistent_http_auth() { ); } -#[cfg(feature = "ilp-over-http")] +#[cfg(all(feature = "sync-sender-tcp", feature = "sync-sender-http"))] #[test] fn cant_use_basic_auth_with_tcp() { let builder = SenderBuilder::new(Protocol::Tcp, "localhost", 9000) @@ -216,7 +224,7 @@ fn cant_use_basic_auth_with_tcp() { ); } -#[cfg(feature = "ilp-over-http")] +#[cfg(all(feature = "sync-sender-tcp", feature = "sync-sender-http"))] #[test] fn cant_use_token_auth_with_tcp() { let builder = SenderBuilder::new(Protocol::Tcp, "localhost", 9000) @@ -228,7 +236,7 @@ fn cant_use_token_auth_with_tcp() { ); } -#[cfg(feature = "ilp-over-http")] +#[cfg(all(feature = "sync-sender-tcp", feature = "sync-sender-http"))] #[test] fn cant_use_ecdsa_auth_with_http() { let builder = SenderBuilder::from_conf("http::addr=localhost;") @@ -247,6 +255,30 @@ fn cant_use_ecdsa_auth_with_http() { ); } +#[cfg(all(not(feature = "sync-sender-tcp"), feature = "sync-sender-http"))] +#[test] +fn cant_use_ecdsa_auth_with_http_ex_tcp_support() { + let mk_builder = || { + SenderBuilder::from_conf("http::addr=localhost;") + .unwrap() + .username("key_id123") + .unwrap() + .token("priv_key123") + .unwrap() + }; + + assert_conf_err( + mk_builder().token_x("pub_key1"), + "cannot specify \"token_x\": ECDSA authentication is only available with ILP/TCP and not available with ILP/HTTP.", + ); + + assert_conf_err( + mk_builder().token_y("pub_key2"), + "cannot specify \"token_y\": ECDSA authentication is only available with ILP/TCP and not available with ILP/HTTP.", + ); +} + +#[cfg(feature = "sync-sender-tcp")] #[test] fn set_auth_specifies_tcp() { let mut builder = SenderBuilder::new(Protocol::Tcp, "localhost", 9000); @@ -263,6 +295,7 @@ fn set_auth_specifies_tcp() { assert_eq!(builder.protocol, Protocol::Tcp); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn set_net_interface_specifies_tcp() { let builder = SenderBuilder::new(Protocol::Tcp, "localhost", 9000); @@ -270,6 +303,7 @@ fn set_net_interface_specifies_tcp() { builder.bind_interface("55.88.0.4").unwrap(); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn tcp_ecdsa_auth() { let builder = SenderBuilder::from_conf( @@ -278,7 +312,7 @@ fn tcp_ecdsa_auth() { .unwrap(); let auth = builder.build_auth().unwrap(); match auth.unwrap() { - AuthParams::Ecdsa(EcdsaAuthParams { + conf::AuthParams::Ecdsa(conf::EcdsaAuthParams { key_id, priv_key, pub_key_x, @@ -289,13 +323,14 @@ fn tcp_ecdsa_auth() { assert_eq!(pub_key_x, "xtok123"); assert_eq!(pub_key_y, "ytok123"); } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] _ => { panic!("Expected AuthParams::Ecdsa"); } } } +#[cfg(feature = "sync-sender-tcp")] #[test] fn incomplete_tcp_ecdsa_auth() { let expected_err_msg = r##"Incomplete ECDSA authentication parameters. Specify either all or none of: "username", "token", "token_x", "token_y"."##; @@ -321,6 +356,7 @@ fn incomplete_tcp_ecdsa_auth() { ); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn misspelled_tcp_ecdsa_auth() { assert_conf_err( @@ -329,6 +365,7 @@ fn misspelled_tcp_ecdsa_auth() { ); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn tcps_tls_verify_on() { let builder = SenderBuilder::from_conf("tcps::addr=localhost;tls_verify=on;").unwrap(); @@ -341,6 +378,7 @@ fn tcps_tls_verify_on() { assert_defaulted_eq(&builder.tls_ca, CertificateAuthority::OsRoots); } +#[cfg(feature = "sync-sender-tcp")] #[cfg(feature = "insecure-skip-verify")] #[test] fn tcps_tls_verify_unsafe_off() { @@ -350,6 +388,7 @@ fn tcps_tls_verify_unsafe_off() { assert_specified_eq(&builder.tls_verify, false); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn tcps_tls_verify_invalid() { assert_conf_err( @@ -358,6 +397,7 @@ fn tcps_tls_verify_invalid() { ); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn tcps_tls_roots_webpki() { let builder = SenderBuilder::from_conf("tcps::addr=localhost;tls_ca=webpki_roots;"); @@ -377,6 +417,7 @@ fn tcps_tls_roots_webpki() { ); } +#[cfg(feature = "sync-sender-tcp")] #[cfg(feature = "tls-native-certs")] #[test] fn tcps_tls_roots_os() { @@ -386,8 +427,11 @@ fn tcps_tls_roots_os() { assert_defaulted_eq(&builder.tls_roots, None); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn tcps_tls_roots_file() { + use std::io::Write; + // Write a dummy file to test the file path let tmp_dir = TempDir::new().unwrap(); let path = tmp_dir.path().join("cacerts.pem"); @@ -402,6 +446,7 @@ fn tcps_tls_roots_file() { assert_specified_eq(&builder.tls_roots, path); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn tcps_tls_roots_file_missing() { let err = @@ -413,8 +458,11 @@ fn tcps_tls_roots_file_missing() { .contains("Could not open root certificate file from path")); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn tcps_tls_roots_file_with_password() { + use std::io::Write; + let tmp_dir = TempDir::new().unwrap(); let path = tmp_dir.path().join("cacerts.pem"); let mut file = std::fs::File::create(&path).unwrap(); @@ -426,7 +474,7 @@ fn tcps_tls_roots_file_with_password() { assert_conf_err(builder_or_err, "\"tls_roots_password\" is not supported."); } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] #[test] fn http_request_min_throughput() { let builder = @@ -439,7 +487,7 @@ fn http_request_min_throughput() { assert_defaulted_eq(&http_config.retry_timeout, Duration::from_millis(10000)); } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] #[test] fn http_request_timeout() { let builder = SenderBuilder::from_conf("http::addr=localhost;request_timeout=100;").unwrap(); @@ -451,7 +499,7 @@ fn http_request_timeout() { assert_defaulted_eq(&http_config.retry_timeout, Duration::from_millis(10000)); } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] #[test] fn http_retry_timeout() { let builder = SenderBuilder::from_conf("http::addr=localhost;retry_timeout=100;").unwrap(); @@ -463,7 +511,7 @@ fn http_retry_timeout() { assert_specified_eq(&http_config.retry_timeout, Duration::from_millis(100)); } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] #[test] fn connect_timeout_uses_request_timeout() { use std::time::Instant; @@ -492,11 +540,13 @@ fn connect_timeout_uses_request_timeout() { assert!(Instant::now() - start < Duration::from_secs(10)); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn auto_flush_off() { SenderBuilder::from_conf("tcps::addr=localhost;auto_flush=off;").unwrap(); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn auto_flush_unsupported() { assert_conf_err( @@ -506,6 +556,7 @@ fn auto_flush_unsupported() { ); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn auto_flush_rows_unsupported() { assert_conf_err( @@ -514,6 +565,7 @@ fn auto_flush_rows_unsupported() { ); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn auto_flush_bytes_unsupported() { assert_conf_err( diff --git a/questdb-rs/src/ingress/tls.rs b/questdb-rs/src/ingress/tls.rs new file mode 100644 index 00000000..85a84fd9 --- /dev/null +++ b/questdb-rs/src/ingress/tls.rs @@ -0,0 +1,247 @@ +use crate::error::{fmt, Result}; +use crate::ingress::CertificateAuthority; +use rustls::RootCertStore; +use rustls_pki_types::CertificateDer; +use std::fs::File; +use std::io::BufReader; +use std::path::Path; +use std::sync::Arc; + +#[cfg(feature = "insecure-skip-verify")] +mod danger { + use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; + use rustls::{DigitallySignedStruct, Error, SignatureScheme}; + use rustls_pki_types::{CertificateDer, ServerName, UnixTime}; + + #[derive(Debug)] + pub struct NoCertificateVerification {} + + impl ServerCertVerifier for NoCertificateVerification { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer<'_>, + _dss: &DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + #[cfg(feature = "aws-lc-crypto")] + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::aws_lc_rs::default_provider() + .signature_verification_algorithms + .supported_schemes() + } + + #[cfg(feature = "ring-crypto")] + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::ring::default_provider() + .signature_verification_algorithms + .supported_schemes() + } + } +} + +#[cfg(feature = "tls-webpki-certs")] +fn add_webpki_roots(root_store: &mut RootCertStore) { + root_store + .roots + .extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()) +} + +#[cfg(feature = "tls-native-certs")] +fn unpack_os_native_certs( + res: rustls_native_certs::CertificateResult, +) -> crate::Result>> { + if !res.errors.is_empty() { + return Err(fmt!( + TlsError, + "Could not load OS native TLS certificates: {}", + res.errors + .iter() + .map(|e| e.to_string()) + .collect::>() + .join(", ") + )); + } + + Ok(res.certs) +} + +#[cfg(feature = "tls-native-certs")] +fn add_os_roots(root_store: &mut RootCertStore) -> crate::Result<()> { + let os_certs = unpack_os_native_certs(rustls_native_certs::load_native_certs())?; + + let (valid_count, invalid_count) = root_store.add_parsable_certificates(os_certs); + if valid_count == 0 && invalid_count > 0 { + return Err(fmt!( + TlsError, + "No valid certificates found in native root store ({} found but were invalid)", + invalid_count + )); + } + Ok(()) +} + +#[derive(Debug)] +pub(crate) enum TlsSettings { + #[cfg(feature = "insecure-skip-verify")] + SkipVerify, + + #[cfg(feature = "tls-webpki-certs")] + WebpkiRoots, + + #[cfg(feature = "tls-native-certs")] + OsRoots, + + #[cfg(all(feature = "tls-webpki-certs", feature = "tls-native-certs"))] + WebpkiAndOsRoots, + + PemFile(Vec>), +} + +impl TlsSettings { + pub fn build( + enabled: bool, + + #[cfg(feature = "insecure-skip-verify")] verify_hostname: bool, + + ca: CertificateAuthority, + roots: Option<&Path>, + ) -> Result> { + if !enabled { + return Ok(None); + } + + #[cfg(feature = "insecure-skip-verify")] + if !verify_hostname { + return Ok(Some(TlsSettings::SkipVerify)); + } + + Ok(Some(match (ca, roots) { + #[cfg(feature = "tls-webpki-certs")] + (CertificateAuthority::WebpkiRoots, None) => TlsSettings::WebpkiRoots, + + #[cfg(feature = "tls-webpki-certs")] + (CertificateAuthority::WebpkiRoots, Some(_)) => { + return Err(fmt!(ConfigError, "Config parameter \"tls_roots\" must be unset when \"tls_ca\" is set to \"webpki_roots\".")); + } + + #[cfg(feature = "tls-native-certs")] + (CertificateAuthority::OsRoots, None) => TlsSettings::OsRoots, + + #[cfg(feature = "tls-native-certs")] + (CertificateAuthority::OsRoots, Some(_)) => { + return Err(fmt!(ConfigError, "Config parameter \"tls_roots\" must be unset when \"tls_ca\" is set to \"os_roots\".")); + } + + #[cfg(all(feature = "tls-webpki-certs", feature = "tls-native-certs"))] + (CertificateAuthority::WebpkiAndOsRoots, None) => TlsSettings::WebpkiAndOsRoots, + + #[cfg(all(feature = "tls-webpki-certs", feature = "tls-native-certs"))] + (CertificateAuthority::WebpkiAndOsRoots, Some(_)) => { + return Err(fmt!(ConfigError, "Config parameter \"tls_roots\" must be unset when \"tls_ca\" is set to \"webpki_and_os_roots\".")); + } + + (CertificateAuthority::PemFile, None) => { + return Err(fmt!(ConfigError, "Config parameter \"tls_roots\" is required when \"tls_ca\" is set to \"pem_file\".")); + } + + (CertificateAuthority::PemFile, Some(pem_file)) => { + let certfile = File::open(pem_file).map_err(|io_err| { + fmt!( + TlsError, + concat!( + "Could not open tls_roots certificate authority ", + "file from path {:?}: {}" + ), + pem_file, + io_err + ) + })?; + let mut reader = BufReader::new(certfile); + let der_certs = rustls_pemfile::certs(&mut reader) + .collect::, _>>() + .map_err(|io_err| { + fmt!( + TlsError, + concat!( + "Could not read certificate authority ", + "file from path {:?}: {}" + ), + pem_file, + io_err + ) + })?; + TlsSettings::PemFile(der_certs) + } + })) + } +} + +pub(crate) fn configure_tls(tls: TlsSettings) -> Result> { + let mut root_store = RootCertStore::empty(); + + #[cfg(feature = "insecure-skip-verify")] + let mut verify_hostname = true; + + match tls { + #[cfg(feature = "tls-webpki-certs")] + TlsSettings::WebpkiRoots => add_webpki_roots(&mut root_store), + + #[cfg(feature = "tls-native-certs")] + TlsSettings::OsRoots => add_os_roots(&mut root_store)?, + + #[cfg(all(feature = "tls-webpki-certs", feature = "tls-native-certs"))] + TlsSettings::WebpkiAndOsRoots => { + add_webpki_roots(&mut root_store); + add_os_roots(&mut root_store)?; + } + + TlsSettings::PemFile(der_certs) => { + root_store.add_parsable_certificates(der_certs); + } + + #[cfg(feature = "insecure-skip-verify")] + TlsSettings::SkipVerify => { + verify_hostname = false; + } + } + + let mut config = rustls::ClientConfig::builder() + .with_root_certificates(root_store) + .with_no_client_auth(); + + // TLS log file for debugging. + // Set the SSLKEYLOGFILE env variable to a writable location. + config.key_log = Arc::new(rustls::KeyLogFile::new()); + + #[cfg(feature = "insecure-skip-verify")] + if !verify_hostname { + config + .dangerous() + .set_certificate_verifier(Arc::new(danger::NoCertificateVerification {})); + } + + Ok(Arc::new(config)) +} diff --git a/questdb-rs/src/lib.rs b/questdb-rs/src/lib.rs index d46a5333..954d2cb1 100644 --- a/questdb-rs/src/lib.rs +++ b/questdb-rs/src/lib.rs @@ -24,7 +24,10 @@ #![doc = include_str!("../README.md")] mod error; + +#[cfg(feature = "sync-sender-tcp")] mod gai; + pub mod ingress; pub use error::*; diff --git a/questdb-rs/src/tests/mock.rs b/questdb-rs/src/tests/mock.rs index d9fba08e..91880fb4 100644 --- a/questdb-rs/src/tests/mock.rs +++ b/questdb-rs/src/tests/mock.rs @@ -36,11 +36,11 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::Instant; -use crate::ingress; -#[cfg(feature = "ilp-over-http")] -use std::io::Write; +#[cfg(feature = "sync-sender-tcp")] +use crate::tests::ndarr::ArrayColumnTypeTag; -use super::ndarr::ArrayColumnTypeTag; +#[cfg(feature = "sync-sender-http")] +use std::io::Write; const CLIENT: Token = Token(0); @@ -53,8 +53,11 @@ pub struct MockServer { tls_conn: Option, pub host: &'static str, pub port: u16, + + #[cfg(feature = "sync-sender-tcp")] pub msgs: Vec>, - #[cfg(feature = "ilp-over-http")] + + #[cfg(feature = "sync-sender-http")] settings_response: serde_json::Value, } @@ -84,7 +87,7 @@ fn tls_config() -> Arc { Arc::new(config) } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] pub struct HttpRequest { method: String, path: String, @@ -92,7 +95,7 @@ pub struct HttpRequest { body: Vec, } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] impl HttpRequest { pub fn method(&self) -> &str { &self.method @@ -111,7 +114,7 @@ impl HttpRequest { } } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] pub struct HttpResponse { status_code: u16, status_text: String, @@ -119,7 +122,7 @@ pub struct HttpResponse { body: Vec, } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] impl HttpResponse { pub fn empty() -> Self { HttpResponse { @@ -181,14 +184,14 @@ impl HttpResponse { } } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] fn contains(haystack: &[u8], needle: &[u8]) -> bool { haystack .windows(needle.len()) .any(|window| window == needle) } -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] fn position(haystack: &[u8], needle: &[u8]) -> Option { haystack .windows(needle.len()) @@ -210,8 +213,11 @@ impl MockServer { tls_conn: None, host: "localhost", port, + + #[cfg(feature = "sync-sender-tcp")] msgs: Vec::new(), - #[cfg(feature = "ilp-over-http")] + + #[cfg(feature = "sync-sender-http")] settings_response: serde_json::Value::Null, }) } @@ -258,6 +264,7 @@ impl MockServer { Ok(()) } + #[cfg(feature = "sync-sender-tcp")] pub fn accept_tls(mut self) -> std::thread::JoinHandle> { std::thread::spawn(|| { self.accept_tls_sync()?; @@ -294,7 +301,7 @@ impl MockServer { self.wait_for(timeout, |event| event.is_readable()) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] pub fn wait_for_send(&mut self, duration: Option) -> io::Result { self.wait_for(duration, |event| event.is_writable()) } @@ -309,7 +316,7 @@ impl MockServer { } } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] pub fn configure_settings_response( mut self, supported_versions: &[u16], @@ -332,7 +339,7 @@ impl MockServer { self } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] fn do_write(&mut self, buf: &[u8]) -> io::Result { let client = self.client.as_mut().unwrap(); if let Some(tls_conn) = self.tls_conn.as_mut() { @@ -343,7 +350,7 @@ impl MockServer { } } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] fn do_write_all(&mut self, buf: &[u8], timeout_sec: Option) -> io::Result<()> { let deadline = timeout_sec.map(|sec| Instant::now() + Duration::from_secs_f64(sec)); let mut pos = 0usize; @@ -378,7 +385,7 @@ impl MockServer { Ok(()) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] fn read_more(&mut self, accum: &mut Vec, deadline: Instant, stage: &str) -> io::Result<()> { let mut chunk = [0u8; 1024]; let count = match self.do_read(&mut chunk[..]) { @@ -415,7 +422,7 @@ impl MockServer { Ok(()) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] fn recv_http_method( &mut self, accum: &mut Vec, @@ -444,7 +451,7 @@ impl MockServer { Ok((body_start, method, path)) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] fn recv_http_headers( &mut self, pos: usize, @@ -473,7 +480,7 @@ impl MockServer { Ok((body_start, headers)) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] pub fn send_http_response( &mut self, response: HttpResponse, @@ -483,7 +490,7 @@ impl MockServer { Ok(()) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] pub fn send_settings_response(&mut self) -> io::Result<()> { let response = HttpResponse::empty() .with_status(200, "OK") @@ -492,12 +499,12 @@ impl MockServer { Ok(()) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] pub fn send_http_response_q(&mut self, response: HttpResponse) -> io::Result<()> { self.send_http_response(response, Some(5.0)) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] pub fn recv_http(&mut self, wait_timeout_sec: f64) -> io::Result { let mut accum = Vec::::new(); let deadline = Instant::now() + Duration::from_secs_f64(wait_timeout_sec); @@ -530,11 +537,12 @@ impl MockServer { }) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] pub fn recv_http_q(&mut self) -> io::Result { self.recv_http(5.0) } + #[cfg(feature = "sync-sender-tcp")] pub fn recv(&mut self, wait_timeout_sec: f64) -> io::Result { let deadline = Instant::now() + Duration::from_secs_f64(wait_timeout_sec); @@ -577,9 +585,9 @@ impl MockServer { index += 1; // calc binary length let binary_type = accum[index]; - if binary_type == ingress::DOUBLE_BINARY_FORMAT_TYPE { + if binary_type == crate::ingress::DOUBLE_BINARY_FORMAT_TYPE { index += size_of::() + 1; - } else if binary_type == ingress::ARRAY_BINARY_FORMAT_TYPE { + } else if binary_type == crate::ingress::ARRAY_BINARY_FORMAT_TYPE { index += 1; let element_type = match ArrayColumnTypeTag::try_from(accum[index]) { Ok(t) => t, @@ -613,24 +621,27 @@ impl MockServer { Ok(received_count) } + #[cfg(feature = "sync-sender-tcp")] pub fn recv_q(&mut self) -> io::Result { self.recv(0.1) } + #[cfg(feature = "sync-sender-tcp")] pub fn lsb_tcp(&self) -> SenderBuilder { SenderBuilder::new(Protocol::Tcp, self.host, self.port) } + #[cfg(feature = "sync-sender-tcp")] pub fn lsb_tcps(&self) -> SenderBuilder { SenderBuilder::new(Protocol::Tcps, self.host, self.port) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] pub fn lsb_http(&self) -> SenderBuilder { SenderBuilder::new(Protocol::Http, self.host, self.port) } - #[cfg(feature = "ilp-over-http")] + #[cfg(feature = "sync-sender-http")] pub fn lsb_https(&self) -> SenderBuilder { SenderBuilder::new(Protocol::Https, self.host, self.port) } diff --git a/questdb-rs/src/tests/mod.rs b/questdb-rs/src/tests/mod.rs index 6bbf55b7..5611c74f 100644 --- a/questdb-rs/src/tests/mod.rs +++ b/questdb-rs/src/tests/mod.rs @@ -24,7 +24,7 @@ mod f64_serializer; -#[cfg(feature = "ilp-over-http")] +#[cfg(feature = "sync-sender-http")] mod http; mod mock; diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 07ddf720..888b2dae 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -24,27 +24,36 @@ use crate::{ ingress::{ - Buffer, CertificateAuthority, Sender, TableName, Timestamp, TimestampMicros, TimestampNanos, + Buffer, F64Serializer, Sender, TableName, Timestamp, TimestampMicros, TimestampNanos, + DOUBLE_BINARY_FORMAT_TYPE, }, - tests::assert_err_contains, - Error, ErrorCode, + ErrorCode, }; -use crate::ingress; +use crate::ingress::ProtocolVersion; +use crate::tests::TestResult; +use core::time::Duration; + #[cfg(feature = "ndarray")] use crate::ingress::ndarr::write_array_data; -use crate::ingress::ProtocolVersion; + +#[cfg(feature = "ndarray")] +use ndarray::{arr2, ArrayD}; + +#[cfg(feature = "sync-sender-tcp")] use crate::tests::{ + assert_err_contains, mock::{certs_dir, MockServer}, ndarr::ArrayColumnTypeTag, - TestResult, }; -use core::time::Duration; -#[cfg(feature = "ndarray")] -use ndarray::{arr2, ArrayD}; + +#[cfg(feature = "sync-sender-tcp")] use rstest::rstest; -use std::io; +#[cfg(feature = "sync-sender-tcp")] +use crate::ingress::{CertificateAuthority, ARRAY_BINARY_FORMAT_TYPE}; + +#[cfg(feature = "sync-sender-tcp")] #[rstest] fn test_basics( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, @@ -99,6 +108,7 @@ fn test_basics( Ok(()) } +#[cfg(feature = "sync-sender-tcp")] #[test] fn test_array_f64_basic() -> TestResult { let mut server = MockServer::new()?; @@ -124,7 +134,7 @@ fn test_array_f64_basic() -> TestResult { f64_to_bytes("f1", 25.5, ProtocolVersion::V2).as_slice(), b",arr1d=", b"=", // binary field - &[ingress::ARRAY_BINARY_FORMAT_TYPE], + &[ARRAY_BINARY_FORMAT_TYPE], &[ArrayColumnTypeTag::Double.into()], &[1u8], // 1D array &3u32.to_le_bytes(), // 3 elements @@ -145,6 +155,7 @@ fn test_array_f64_basic() -> TestResult { Ok(()) } +#[cfg(feature = "sync-sender-tcp")] #[cfg(feature = "ndarray")] #[test] fn test_array_f64_for_ndarray() -> TestResult { @@ -172,7 +183,7 @@ fn test_array_f64_for_ndarray() -> TestResult { let array_header2d = &[ &[b'='][..], - &[ingress::ARRAY_BINARY_FORMAT_TYPE], + &[ARRAY_BINARY_FORMAT_TYPE], &[ArrayColumnTypeTag::Double.into()], &[2u8], &2i32.to_le_bytes(), @@ -184,7 +195,7 @@ fn test_array_f64_for_ndarray() -> TestResult { let array_header3d = &[ &[b'='][..], - &[ingress::ARRAY_BINARY_FORMAT_TYPE], + &[ARRAY_BINARY_FORMAT_TYPE], &[ArrayColumnTypeTag::Double.into()], &[3u8], &2i32.to_le_bytes(), @@ -222,6 +233,7 @@ fn test_array_f64_for_ndarray() -> TestResult { Ok(()) } +#[cfg(feature = "sync-sender-tcp")] #[rstest] fn test_max_buf_size( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, @@ -379,6 +391,7 @@ fn test_transactional() -> TestResult { Ok(()) } +#[cfg(feature = "sync-sender-tcp")] #[test] fn test_auth_inconsistent_keys() -> TestResult { test_bad_key("fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU", // d @@ -388,6 +401,7 @@ fn test_auth_inconsistent_keys() -> TestResult { ) } +#[cfg(feature = "sync-sender-tcp")] #[test] fn test_auth_bad_base64_private_key() -> TestResult { test_bad_key( @@ -398,6 +412,7 @@ fn test_auth_bad_base64_private_key() -> TestResult { ) } +#[cfg(feature = "sync-sender-tcp")] #[test] fn test_auth_private_key_too_long() -> TestResult { #[cfg(feature = "aws-lc-crypto")] @@ -414,6 +429,7 @@ fn test_auth_private_key_too_long() -> TestResult { ) } +#[cfg(feature = "sync-sender-tcp")] #[test] fn test_auth_public_key_x_too_long() -> TestResult { test_bad_key( @@ -424,6 +440,7 @@ fn test_auth_public_key_x_too_long() -> TestResult { ) } +#[cfg(feature = "sync-sender-tcp")] #[test] fn test_auth_public_key_y_too_long() -> TestResult { test_bad_key( @@ -434,6 +451,7 @@ fn test_auth_public_key_y_too_long() -> TestResult { ) } +#[cfg(feature = "sync-sender-tcp")] #[test] fn test_auth_bad_base64_public_key_x() -> TestResult { test_bad_key( @@ -444,6 +462,7 @@ fn test_auth_bad_base64_public_key_x() -> TestResult { ) } +#[cfg(feature = "sync-sender-tcp")] #[test] fn test_auth_bad_base64_public_key_y() -> TestResult { test_bad_key( @@ -454,6 +473,7 @@ fn test_auth_bad_base64_public_key_y() -> TestResult { ) } +#[cfg(feature = "sync-sender-tcp")] fn test_bad_key( priv_key: &str, pub_key_x: &str, @@ -591,6 +611,7 @@ fn test_arr_column_name_too_long() -> TestResult { column_name_too_long_test_impl!(column_arr, &[1.0, 2.0, 3.0]) } +#[cfg(feature = "sync-sender-tcp")] #[rstest] fn test_tls_with_file_ca( #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, @@ -629,6 +650,7 @@ fn test_tls_with_file_ca( Ok(()) } +#[cfg(feature = "sync-sender-tcp")] #[test] fn test_tls_to_plain_server() -> TestResult { let mut ca_path = certs_dir(); @@ -640,7 +662,7 @@ fn test_tls_to_plain_server() -> TestResult { .auth_timeout(Duration::from_millis(500))? .tls_ca(CertificateAuthority::PemFile)? .tls_roots(ca_path)?; - let server_jh = std::thread::spawn(move || -> io::Result { + let server_jh = std::thread::spawn(move || -> std::io::Result { server.accept()?; Ok(server) }); @@ -649,7 +671,7 @@ fn test_tls_to_plain_server() -> TestResult { let err = maybe_sender.unwrap_err(); assert_eq!( err, - Error::new( + crate::error::Error::new( ErrorCode::TlsError, "Failed to complete TLS handshake: \ Timed out waiting for server response after 500ms." @@ -659,6 +681,7 @@ fn test_tls_to_plain_server() -> TestResult { Ok(()) } +#[cfg(feature = "sync-sender-tcp")] fn expect_eventual_disconnect(sender: &mut Sender) { let mut retry = || { for _ in 0..1000 { @@ -670,10 +693,11 @@ fn expect_eventual_disconnect(sender: &mut Sender) { Ok(()) }; - let err: Error = retry().unwrap_err(); + let err: crate::error::Error = retry().unwrap_err(); assert_eq!(err.code(), ErrorCode::SocketError); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn test_plain_to_tls_server() -> TestResult { let server = MockServer::new()?; @@ -684,8 +708,8 @@ fn test_plain_to_tls_server() -> TestResult { // The server failed to handshake, so disconnected the client. assert!( - (server_err.kind() == io::ErrorKind::TimedOut) - || (server_err.kind() == io::ErrorKind::WouldBlock) + (server_err.kind() == std::io::ErrorKind::TimedOut) + || (server_err.kind() == std::io::ErrorKind::WouldBlock) ); // The client nevertheless connected successfully. @@ -741,6 +765,7 @@ fn bad_uppercase_protocol() { assert!(err.msg() == "Unsupported protocol: TCP"); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn bad_uppercase_addr() { let res = Sender::from_conf("tcp::ADDR=localhost:9009;"); @@ -750,6 +775,7 @@ fn bad_uppercase_addr() { assert!(err.msg() == "Missing \"addr\" parameter in config string"); } +#[cfg(feature = "sync-sender-tcp")] #[test] fn tcp_mismatched_buffer_and_sender_version() -> TestResult { let server = MockServer::new()?; @@ -775,12 +801,12 @@ pub(crate) fn f64_to_bytes(name: &str, value: f64, version: ProtocolVersion) -> match version { ProtocolVersion::V1 => { - let mut ser = crate::ingress::F64Serializer::new(value); + let mut ser = F64Serializer::new(value); buf.extend_from_slice(ser.as_str().as_bytes()); } ProtocolVersion::V2 => { buf.push(b'='); - buf.push(crate::ingress::DOUBLE_BINARY_FORMAT_TYPE); + buf.push(DOUBLE_BINARY_FORMAT_TYPE); buf.extend_from_slice(&value.to_le_bytes()); } } diff --git a/system_test/fixture.py b/system_test/fixture.py index 1f70da13..b6b95d3c 100644 --- a/system_test/fixture.py +++ b/system_test/fixture.py @@ -40,6 +40,8 @@ import urllib.request import urllib.parse import urllib.error +import concurrent.futures +import threading from pprint import pformat AUTH_TXT = """admin ec-p-256-sha256 fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU Dt5tbS1dEDMSYfym3fgMv0B99szno-dFc1rYF9t0aac @@ -229,7 +231,122 @@ class QueryError(Exception): pass -class QuestDbFixture: +class QuestDbFixtureBase: + def print_log(self): + """Print the QuestDB log to stderr.""" + sys.stderr.write('questdb log output skipped.\n') + + def http_sql_query(self, sql_query): + url = ( + f'http://{self.host}:{self.http_server_port}/exec?' + + urllib.parse.urlencode({'query': sql_query})) + buf = None + try: + resp = urllib.request.urlopen(url, timeout=5) + buf = resp.read() + except urllib.error.HTTPError as http_error: + buf = http_error.read() + try: + data = json.loads(buf) + except json.JSONDecodeError as jde: + # Include buffer in error message for easier debugging. + raise json.JSONDecodeError( + f'Could not parse response: {buf!r}: {jde.msg}', + jde.doc, + jde.pos) + if 'error' in data: + raise QueryError(data['error']) + return data + + def query_version(self): + try: + res = self.http_sql_query('select build') + except QueryError as qe: + # For old versions that don't support `build` yet, parse from path. + return self.version + + vers = res['dataset'][0][0] + print(vers) + + # This returns a string like: + # 'Build Information: QuestDB 7.3.2, JDK 11.0.8, Commit Hash 19059deec7b0fd19c53182b297a5d59774a51892' + # We want the '7.3.2' part. + vers = re.compile(r'.*QuestDB ([0-9.]+).*').search(vers).group(1) + return _parse_version(vers) + + def retry_check_table( + self, + table_name, + *, + min_rows=1, + timeout_sec=300, + log=True, + log_ctx=None): + sql_query = f"select * from '{table_name}'" + http_response_log = [] + + def check_table(): + try: + resp = self.http_sql_query(sql_query) + http_response_log.append((time.time(), resp)) + if not resp.get('dataset'): + return False + elif len(resp['dataset']) < min_rows: + return False + return resp + except QueryError: + return None + + try: + return retry(check_table, timeout_sec=timeout_sec) + except TimeoutError as toe: + if log: + if log_ctx: + log_ctx_str = log_ctx.decode('utf-8', errors='replace') + log_ctx = f'\n{textwrap.indent(log_ctx_str, " ")}\n' + sys.stderr.write( + f'Timed out after {timeout_sec} seconds ' + + f'waiting for query {sql_query!r}. ' + + f'Context: {log_ctx}' + + f'Client response log:\n' + + pformat(http_response_log) + + f'\nQuestDB log:\n') + self.print_log() + raise toe + + def show_tables(self): + """Return a list of tables in the database.""" + sql_query = "show tables" + try: + resp = self.http_sql_query(sql_query) + return [row[0] for row in resp['dataset']] + except QueryError as qe: + raise qe + + def drop_table(self, table_name): + self.http_sql_query(f"drop table '{table_name}'") + + def drop_all_tables(self): + """Drop all tables in the database.""" + all_tables = self.show_tables() + # if all_tables: + # print(f'Dropping {len(all_tables)} tables: {all_tables!r}') + for table_name in all_tables: + self.drop_table(table_name) + + +class QuestDbExternalFixture(QuestDbFixtureBase): + def __init__(self, host, line_tcp_port, http_server_port, version, http, auth, protocol_version): + self.host = host + self.line_tcp_port = line_tcp_port + self.http_server_port = http_server_port + self.version = version + self.http = http + self.auth = auth + self.protocol_version = protocol_version + + +class QuestDbFixture(QuestDbFixtureBase): def __init__(self, root_dir: pathlib.Path, auth=False, wrap_tls=False, http=False, protocol_version=None): self._root_dir = root_dir self.version = _parse_version(self._root_dir.name) @@ -347,104 +464,6 @@ def check_http_up(): self._tls_proxy.start() self.tls_line_tcp_port = self._tls_proxy.listen_port - def http_sql_query(self, sql_query): - url = ( - f'http://{self.host}:{self.http_server_port}/exec?' + - urllib.parse.urlencode({'query': sql_query})) - buf = None - try: - resp = urllib.request.urlopen(url, timeout=5) - buf = resp.read() - except urllib.error.HTTPError as http_error: - buf = http_error.read() - try: - data = json.loads(buf) - except json.JSONDecodeError as jde: - # Include buffer in error message for easier debugging. - raise json.JSONDecodeError( - f'Could not parse response: {buf!r}: {jde.msg}', - jde.doc, - jde.pos) - if 'error' in data: - raise QueryError(data['error']) - return data - - def query_version(self): - try: - res = self.http_sql_query('select build') - except QueryError as qe: - # For old versions that don't support `build` yet, parse from path. - return self.version - - vers = res['dataset'][0][0] - print(vers) - - # This returns a string like: - # 'Build Information: QuestDB 7.3.2, JDK 11.0.8, Commit Hash 19059deec7b0fd19c53182b297a5d59774a51892' - # We want the '7.3.2' part. - vers = re.compile(r'.*QuestDB ([0-9.]+).*').search(vers).group(1) - return _parse_version(vers) - - def retry_check_table( - self, - table_name, - *, - min_rows=1, - timeout_sec=300, - log=True, - log_ctx=None): - sql_query = f"select * from '{table_name}'" - http_response_log = [] - - def check_table(): - try: - resp = self.http_sql_query(sql_query) - http_response_log.append((time.time(), resp)) - if not resp.get('dataset'): - return False - elif len(resp['dataset']) < min_rows: - return False - return resp - except QueryError: - return None - - try: - return retry(check_table, timeout_sec=timeout_sec) - except TimeoutError as toe: - if log: - if log_ctx: - log_ctx_str = log_ctx.decode('utf-8', errors='replace') - log_ctx = f'\n{textwrap.indent(log_ctx_str, " ")}\n' - sys.stderr.write( - f'Timed out after {timeout_sec} seconds ' + - f'waiting for query {sql_query!r}. ' + - f'Context: {log_ctx}' + - f'Client response log:\n' + - pformat(http_response_log) + - f'\nQuestDB log:\n') - self.print_log() - raise toe - - def show_tables(self): - """Return a list of tables in the database.""" - sql_query = "show tables" - try: - resp = self.http_sql_query(sql_query) - return [row[0] for row in resp['dataset']] - except QueryError as qe: - raise qe - - def drop_table(self, table_name): - self.http_sql_query(f"drop table '{table_name}'") - - def drop_all_tables(self): - """Drop all tables in the database.""" - all_tables = self.show_tables() - # if all_tables: - # print(f'Dropping {len(all_tables)} tables: {all_tables!r}') - for table_name in all_tables: - self.drop_table(table_name) - def __enter__(self): self.start() @@ -470,18 +489,41 @@ def __init__(self, qdb_ilp_port): proj = Project() self._code_dir = proj.root_dir / 'system_test' / 'tls_proxy' self._target_dir = proj.build_dir / 'tls_proxy' - self._log_path = self._target_dir / 'log.txt' - self._log_file = None self._proc = None + self._port_future = None + + def _capture_output(self, pipe, port_future): + """Capture output from subprocess and forward to stderr while watching for port""" + try: + for line in iter(pipe.readline, b''): + line_str = line.decode('utf-8', errors='replace') + # Write to stderr + sys.stderr.write(line_str) + sys.stderr.flush() + + # Check for port if we haven't found it yet + if not port_future.done(): + listening_msg = '[TLS PROXY] TLS Proxy is listening on localhost:' + if line_str.startswith(listening_msg) and line_str.endswith('.\n'): + port_str = line_str[len(listening_msg):-2] + try: + port = int(port_str) + port_future.set_result(port) + except ValueError: + pass # Invalid port, keep looking + except Exception as e: + if not port_future.done(): + port_future.set_exception(e) + finally: + pipe.close() def start(self): self._target_dir.mkdir(exist_ok=True) env = dict(os.environ) env['CARGO_TARGET_DIR'] = str(self._target_dir) - self._log_file = open(self._log_path, 'wb') # Compile before running `cargo run`. - # Note that errors and output are purpously suppressed. + # Note that errors and output are purposely suppressed. # This is just to exclude the build time from the start-up time. # If there are build errors, they'll be reported later in the `run` # call below. @@ -496,24 +538,24 @@ def start(self): ['cargo', 'run', str(self.qdb_ilp_port)], cwd=self._code_dir, env=env, - stdout=self._log_file, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - def check_started(): - with open(self._log_path, 'r', encoding='utf-8') as log_reader: - lines = log_reader.readlines() - for line in lines: - listening_msg = 'TLS Proxy is listening on localhost:' - if line.startswith(listening_msg) and line.endswith('.\n'): - port_str = line[len(listening_msg):-2] - port = int(port_str) - return port - return None - - self.listen_port = retry( - check_started, - timeout_sec=180, # Longer to include time to compile. - msg='Timed out waiting for `tls_proxy` to start.', ) + # Create future for port detection + self._port_future = concurrent.futures.Future() + + # Start thread to capture and forward output + self._output_thread = threading.Thread( + target=self._capture_output, + args=(self._proc.stdout, self._port_future)) + self._output_thread.daemon = True + self._output_thread.start() + + # Wait for port detection with timeout + try: + self.listen_port = self._port_future.result(timeout=180) + except concurrent.futures.TimeoutError as toe: + raise RuntimeError('Timed out waiting for `tls_proxy` to start.') from toe def connect_to_listening_port(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -532,9 +574,8 @@ def connect_to_listening_port(): def stop(self): if self._proc: + if self._output_thread.is_alive(): + self._output_thread.join(timeout=5) self._proc.terminate() self._proc.wait() self._proc = None - if self._log_file: - self._log_file.close() - self._log_file = None diff --git a/system_test/test.py b/system_test/test.py index f39fab36..9465d630 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -40,6 +40,8 @@ import uuid from fixture import ( Project, + QuestDbFixtureBase, + QuestDbExternalFixture, QuestDbFixture, TlsProxyFixture, install_questdb, @@ -49,7 +51,7 @@ import subprocess from collections import namedtuple -QDB_FIXTURE: QuestDbFixture = None +QDB_FIXTURE: QuestDbFixtureBase = None TLS_PROXY_FIXTURE: TlsProxyFixture = None BUILD_MODE = None @@ -137,6 +139,14 @@ def _expect_eventual_disconnect(self, sender): .at_now()) sender.flush() + def setUp(self): + test_name = self.id() + QDB_FIXTURE.http_sql_query(f'select * from long_sequence(1) -- >>>>>>>>> BEGIN PYTHON UNIT TEST: {test_name}') + + def tearDown(self): + test_name = self.id() + QDB_FIXTURE.http_sql_query(f'select * from long_sequence(1) -- <<<<<<<<< END PYTHON UNIT TEST: {test_name}') + def test_default_max_name_len(self): with self._mk_linesender() as sender: self.assertEqual(sender.max_name_len, 127) @@ -233,15 +243,19 @@ def _test_single_symbol_impl(self, sender): .table(table_name) .symbol('a', 'A') .at_now()) + (sender + .table(table_name) + .symbol('a', 'B') + .at_now()) pending = sender.buffer.peek() - resp = retry_check_table(table_name, log_ctx=pending) + resp = retry_check_table(table_name, log_ctx=pending, min_rows=2) exp_columns = [ {'name': 'a', 'type': 'SYMBOL'}, {'name': 'timestamp', 'type': 'TIMESTAMP'}] self.assertEqual(resp['columns'], exp_columns) - exp_dataset = [['A']] # Comparison excludes timestamp column. + exp_dataset = [['A'], ['B']] # Comparison excludes timestamp column. scrubbed_dataset = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_dataset, exp_dataset) @@ -1119,24 +1133,16 @@ def list_releases(args): def run_with_existing(args): global QDB_FIXTURE - MockFixture = namedtuple( - 'MockFixture', - ( - 'host', - 'line_tcp_port', - 'http_server_port', - 'version', - 'http', - 'auth', - 'protocol_version')) host, line_tcp_port, http_server_port = args.existing.split(':') - QDB_FIXTURE = MockFixture( + QDB_FIXTURE = QuestDbExternalFixture( host, int(line_tcp_port), int(http_server_port), (999, 999, 999), True, - False) + False, + qls.ProtocolVersion.V2 + ) unittest.main() @@ -1186,7 +1192,7 @@ def run_with_fixtures(args): auth=auth) TLS_PROXY_FIXTURE = None try: - print(f'>>>> STARTING {questdb_dir} [auth={auth}] <<<<') + sys.stderr.write(f'>>>> STARTING {questdb_dir} [auth={auth}] <<<<\n') QDB_FIXTURE.start() for http, protocol_version, build_mode in itertools.product( (False, True), # http @@ -1198,8 +1204,8 @@ def run_with_fixtures(args): continue if auth and (protocol_version != latest_protocol): continue - print( - f'Running tests [auth={auth}, http={http}, build_mode={build_mode}, protocol_version={protocol_version}]') + sys.stderr.write( + f'>>>> Running tests [auth={auth}, http={http}, build_mode={build_mode}, protocol_version={protocol_version}]\n') # Read the version _after_ a first start so it can rely # on the live one from the `select build` query. BUILD_MODE = build_mode diff --git a/system_test/tls_proxy/Cargo.lock b/system_test/tls_proxy/Cargo.lock index 2ca51146..512fa4b9 100644 --- a/system_test/tls_proxy/Cargo.lock +++ b/system_test/tls_proxy/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -17,27 +17,37 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "argh" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7af5ba06967ff7214ce4c7419c7d185be7ecd6cc4965a8f6e1d8ce0398aad219" +checksum = "34ff18325c8a36b82f992e533ece1ec9f9a9db446bd1c14d4f936bac88fcd240" dependencies = [ "argh_derive", "argh_shared", + "rust-fuzzy-search", ] [[package]] name = "argh_derive" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56df0aeedf6b7a2fc67d06db35b09684c3e8da0c95f8f27685cb17e08413d87a" +checksum = "adb7b2b83a50d329d5d8ccc620f5c7064028828538bdf5646acd60dc1f767803" dependencies = [ "argh_shared", "proc-macro2", @@ -47,9 +57,9 @@ dependencies = [ [[package]] name = "argh_shared" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5693f39141bda5760ecc4111ab08da40565d1771038c4a0250f03457ec707531" +checksum = "a464143cc82dedcdc3928737445362466b7674b5db4e2eb8e869846d6d84f4f6" dependencies = [ "serde", ] @@ -60,6 +70,29 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-lc-rs" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fcc8f365936c834db5514fc45aee5b1202d677e6b40e48468aaaa8183ca8c7" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61b1d86e7705efe1be1b569bab41d4fa1e14e220b60a160f78de2db687add079" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "backtrace" version = "0.3.69" @@ -76,10 +109,27 @@ dependencies = [ ] [[package]] -name = "base64" -version = "0.21.4" +name = "bindgen" +version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.9.1", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", + "which", +] [[package]] name = "bitflags" @@ -87,19 +137,36 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + [[package]] name = "bytes" -version = "1.1.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cc" -version = "1.0.83" +version = "1.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" dependencies = [ + "jobserver", "libc", + "shlex", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", ] [[package]] @@ -108,11 +175,59 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "errno" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" -version = "0.3.29" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -125,9 +240,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -135,15 +250,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -152,15 +267,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -169,21 +284,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -215,19 +330,82 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] -name = "hermit-abi" -version = "0.1.19" +name = "glob" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" + +[[package]] +name = "home" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags 2.9.1", + "cfg-if", + "libc", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "jobserver" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" -version = "0.2.148" +version = "0.2.174" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" + +[[package]] +name = "libloading" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" +dependencies = [ + "cfg-if", + "windows-targets 0.53.2", +] + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "lock_api" @@ -254,6 +432,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.1" @@ -265,23 +449,23 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] -name = "num_cpus" -version = "1.13.1" +name = "nom" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ - "hermit-abi", - "libc", + "memchr", + "minimal-lexical", ] [[package]] @@ -293,6 +477,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + [[package]] name = "parking_lot" version = "0.12.1" @@ -328,20 +518,30 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "prettyplease" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "061c1221631e079b26479d25bbf2275bfe5917ae8419cd7e34f13bfc2aa7539a" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.33" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] @@ -352,9 +552,38 @@ version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ - "bitflags", + "bitflags 1.3.2", +] + +[[package]] +name = "regex" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + [[package]] name = "ring" version = "0.17.7" @@ -369,20 +598,46 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rust-fuzzy-search" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a157657054ffe556d8858504af8a672a054a6e0bd9e8ee531059100c0fa11bb2" + [[package]] name = "rustc-demangle" version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" +dependencies = [ + "bitflags 2.9.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + [[package]] name = "rustls" -version = "0.22.2" +version = "0.23.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" +checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" dependencies = [ + "aws-lc-rs", "log", - "ring", + "once_cell", "rustls-pki-types", "rustls-webpki", "subtle", @@ -391,26 +646,29 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.0.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.2.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a716eb65e3158e90e17cd93d855216e27bde02745ab842f2cab4a39dba1bacf" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "zeroize", +] [[package]] name = "rustls-webpki" -version = "0.102.2" +version = "0.103.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" +checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -442,6 +700,12 @@ dependencies = [ "syn", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -468,12 +732,12 @@ checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -490,9 +754,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "2.0.33" +version = "2.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9caece70c63bfba29ec2fed841a09851b14a235c60010fa4de58089b6c025668" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" dependencies = [ "proc-macro2", "quote", @@ -514,28 +778,29 @@ dependencies = [ [[package]] name = "tokio" -version = "1.32.0" +version = "1.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "1140bb80481756a8cbe10541f37433b459c5aa1e727b4c020fbfebdc25bf3ec4" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", - "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", + "slab", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", @@ -544,12 +809,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.25.0" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ "rustls", - "rustls-pki-types", "tokio", ] @@ -571,6 +835,18 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + [[package]] name = "windows-sys" version = "0.36.1" @@ -590,7 +866,34 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.2", ] [[package]] @@ -599,21 +902,65 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", + "windows_aarch64_gnullvm 0.48.5", "windows_aarch64_msvc 0.48.5", "windows_i686_gnu 0.48.5", "windows_i686_msvc 0.48.5", "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm", + "windows_x86_64_gnullvm 0.48.5", "windows_x86_64_msvc 0.48.5", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.36.1" @@ -626,6 +973,18 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.36.1" @@ -638,6 +997,30 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.36.1" @@ -650,6 +1033,18 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.36.1" @@ -662,12 +1057,36 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.36.1" @@ -680,6 +1099,18 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "zeroize" version = "1.7.0" diff --git a/system_test/tls_proxy/Cargo.toml b/system_test/tls_proxy/Cargo.toml index 73d72564..303dfbc0 100644 --- a/system_test/tls_proxy/Cargo.toml +++ b/system_test/tls_proxy/Cargo.toml @@ -6,10 +6,10 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tokio = { version = "1.32.0", features = ["full"] } -tokio-rustls = "0.25.0" -rustls = "0.22.2" -rustls-pemfile = "2.0.0" -argh = "0.1.12" -anyhow = "1.0.75" -futures = "0.3.29" \ No newline at end of file +tokio = { version = "1.46.0", features = ["full"] } +tokio-rustls = "0.26.2" +rustls = "0.23.28" +rustls-pemfile = "2.2.0" +argh = "0.1.13" +anyhow = "1.0.98" +futures = "0.3.31" diff --git a/system_test/tls_proxy/src/lib.rs b/system_test/tls_proxy/src/lib.rs index 8dee12c9..2914a373 100644 --- a/system_test/tls_proxy/src/lib.rs +++ b/system_test/tls_proxy/src/lib.rs @@ -68,14 +68,13 @@ async fn handle_conn( acceptor: &TlsAcceptor, dest_addr: &str, ) -> Result<(), Box> { - eprintln!("Waiting for a connection."); let (inbound_conn, _) = listener.accept().await?; - eprintln!("Accepted a client connection."); + inbound_conn.set_nodelay(true)?; + let outbound_conn = TcpStream::connect(dest_addr); let acceptor = acceptor.clone(); let inbound_conn = acceptor.accept(inbound_conn).await?; - eprintln!("Completed TLS handshake with client connection."); - let outbound_conn = TcpStream::connect(dest_addr).await?; - eprintln!("Established outbound connection to {}.", dest_addr); + let outbound_conn = outbound_conn.await?; + outbound_conn.set_nodelay(true)?; let (mut in_read, mut in_write) = tio::split(inbound_conn); let (mut out_read, mut out_write) = outbound_conn.into_split(); @@ -84,8 +83,8 @@ async fn handle_conn( let out_to_in = tokio::spawn(async move { tio::copy(&mut out_read, &mut in_write).await }); select! { - _ = in_to_out => eprintln!("in_to_out shut down."), - _ = out_to_in => eprintln!("out_to_in shut down."), + _ = in_to_out => eprintln!("[TLS PROXY] in_to_out shut down."), + _ = out_to_in => eprintln!("[TLS PROXY] out_to_in shut down."), } Ok(()) @@ -95,20 +94,20 @@ async fn loop_server( dest_port: u16, listen_port_sender: tokio::sync::oneshot::Sender, ) -> anyhow::Result<()> { - let dest_addr = format!("localhost:{}", dest_port); - eprintln!("Destination address is {}.", &dest_addr); + let dest_addr = format!("localhost:{dest_port}"); + eprintln!("[TLS PROXY] Destination address is {}.", &dest_addr); let config = tls_config(); let acceptor = TlsAcceptor::from(config); let listener = TcpListener::bind("0.0.0.0:0").await?; let listen_port = listener.local_addr()?.port(); - eprintln!("TLS Proxy is listening on localhost:{}.", listen_port); + eprintln!("[TLS PROXY] TLS Proxy is listening on localhost:{listen_port}."); listen_port_sender.send(listen_port).unwrap(); loop { if let Err(err) = handle_conn(&listener, &acceptor, &dest_addr).await { - eprintln!("Error handling connection: {}", err); + eprintln!("[TLS PROXY] Error handling connection: {err}"); } } } diff --git a/system_test/tls_proxy/src/main.rs b/system_test/tls_proxy/src/main.rs index 02f85898..48131fa2 100644 --- a/system_test/tls_proxy/src/main.rs +++ b/system_test/tls_proxy/src/main.rs @@ -35,7 +35,7 @@ struct Options { port: u16, } -#[tokio::main] +#[tokio::main(worker_threads = 2)] async fn main() -> anyhow::Result<()> { let options: Options = argh::from_env(); let server = tls_proxy::TlsProxy::new(options.port)?;