diff --git a/core/include/ten_utils/log/log.h b/core/include/ten_utils/log/log.h index e81f754ecc..d8b17f8595 100644 --- a/core/include/ten_utils/log/log.h +++ b/core/include/ten_utils/log/log.h @@ -80,11 +80,63 @@ typedef void (*ten_log_encrypt_on_encrypt_func_t)(uint8_t *data, void *user_data); typedef void (*ten_log_encrypt_on_deinit_func_t)(void *user_data); +/** + * @brief Structure to pass location information to log functions. + * + * This structure contains the location context (app_uri, graph_id, + * extension_name) for a log message. It's designed for efficient FFI passing + * between C and Rust. + * + * @warning CRITICAL: Keep in sync with Rust definition + * + * This struct MUST exactly match the Rust struct `TenLogLocInfo` defined in: + * `core/src/ten_rust/src/log/bindings.rs` + * + * Rust Definition (for reference): + * @code{.rs} + * #[repr(C)] + * pub struct TenLogLocInfo { + * pub app_uri: *const c_char, + * pub app_uri_len: usize, + * pub graph_id: *const c_char, + * pub graph_id_len: usize, + * pub extension_name: *const c_char, + * pub extension_name_len: usize, + * } + * @endcode + * + * @note Memory Layout Requirements: + * - Field order must match exactly + * - Field types must have same size and alignment + * - No padding should be introduced between fields (naturally aligned) + * - On 64-bit: sizeof = 48 bytes, On 32-bit: sizeof = 24 bytes + * + * @note Safety: + * - Modifying this struct requires updating the Rust definition + * - Modifying the Rust definition requires updating this struct + * - Rust has compile-time assertions to verify layout compatibility + * + * @note Verification: + * - Run Rust test: `cargo test test_ten_log_loc_info_layout` + * - The test verifies size, alignment, and field offsets + * + * @see TenLogLocInfo in core/src/ten_rust/src/log/bindings.rs + */ +typedef struct ten_log_loc_info_t { + const char *app_uri; + size_t app_uri_len; + const char *graph_id; + size_t graph_id_len; + const char *extension_name; + size_t extension_name_len; +} ten_log_loc_info_t; + typedef void (*ten_log_advanced_log_func_t)( ten_log_t *self, TEN_LOG_LEVEL level, const char *category, size_t category_len, const char *func_name, size_t func_name_len, const char *file_name, size_t file_name_len, size_t line_no, - const char *msg, size_t msg_len, ten_value_t *fields); + const char *msg, size_t msg_len, ten_value_t *fields, + const ten_log_loc_info_t *loc_info); typedef void (*ten_log_advanced_log_reopen_all_func_t)(ten_log_t *self, void *config); diff --git a/core/include_internal/ten_runtime/common/loc.h b/core/include_internal/ten_runtime/common/loc.h index f6fa52acdf..7440900e69 100644 --- a/core/include_internal/ten_runtime/common/loc.h +++ b/core/include_internal/ten_runtime/common/loc.h @@ -11,6 +11,7 @@ #include #include "ten_utils/lib/string.h" +#include "ten_utils/log/log.h" #include "ten_utils/value/value.h" #define TEN_LOC_SIGNATURE 0x581B639EF70CBC5DU @@ -141,3 +142,7 @@ TEN_RUNTIME_PRIVATE_API void ten_loc_set_extension_name_with_size( TEN_RUNTIME_PRIVATE_API void ten_loc_set_extension_name( ten_loc_t *self, const char *extension_name); + +// Helper function to initialize ten_log_loc_info_t from ten_loc_t. +TEN_RUNTIME_PRIVATE_API void ten_log_loc_info_init_from_loc( + ten_log_loc_info_t *info, ten_loc_t *loc); diff --git a/core/include_internal/ten_runtime/global/log.h b/core/include_internal/ten_runtime/global/log.h index 4940c91fa1..d55328c2e0 100644 --- a/core/include_internal/ten_runtime/global/log.h +++ b/core/include_internal/ten_runtime/global/log.h @@ -23,7 +23,8 @@ TEN_RUNTIME_PRIVATE_API void ten_log_rust_log_func( ten_log_t *self, TEN_LOG_LEVEL level, const char *category, size_t category_len, const char *func_name, size_t func_name_len, const char *file_name, size_t file_name_len, size_t line_no, - const char *msg, size_t msg_len, ten_value_t *fields); + const char *msg, size_t msg_len, ten_value_t *fields, + const ten_log_loc_info_t *loc_info); TEN_RUNTIME_PRIVATE_API void ten_log_rust_config_deinit(void *config); diff --git a/core/include_internal/ten_runtime/ten_env/ten_env.h b/core/include_internal/ten_runtime/ten_env/ten_env.h index 1472ad904a..63f489c61f 100644 --- a/core/include_internal/ten_runtime/ten_env/ten_env.h +++ b/core/include_internal/ten_runtime/ten_env/ten_env.h @@ -18,6 +18,7 @@ typedef struct ten_engine_t ten_engine_t; typedef struct ten_addon_loader_t ten_addon_loader_t; +typedef struct ten_loc_t ten_loc_t; typedef void (*ten_env_destroy_handler_in_target_lang_func_t)( void *me_in_target_lang); @@ -115,6 +116,10 @@ TEN_RUNTIME_PRIVATE_API void ten_env_set_attach_to( TEN_RUNTIME_PRIVATE_API const char *ten_env_get_attached_instance_name( ten_env_t *self, bool check_thread); +TEN_RUNTIME_PRIVATE_API void ten_env_get_attached_target_loc(ten_env_t *self, + ten_loc_t *loc, + bool check_thread); + TEN_RUNTIME_PRIVATE_API ten_app_t *ten_env_get_belonging_app(ten_env_t *self); inline ten_extension_t *ten_env_get_attached_extension(ten_env_t *self) { diff --git a/core/include_internal/ten_rust/ten_rust.h b/core/include_internal/ten_rust/ten_rust.h index 70b3f1df02..f10ba0342e 100644 --- a/core/include_internal/ten_rust/ten_rust.h +++ b/core/include_internal/ten_rust/ten_rust.h @@ -229,7 +229,8 @@ TEN_RUST_PRIVATE_API void ten_rust_log( AdvancedLogConfig *config, const char *category, size_t category_len, int64_t pid, int64_t tid, int level, const char *func_name, size_t func_name_len, const char *file_name, size_t file_name_len, - size_t line_no, const char *msg, size_t msg_len); + size_t line_no, const ten_log_loc_info_t *loc_info, const char *msg, + size_t msg_len); TEN_RUST_PRIVATE_API void ten_rust_log_config_destroy( AdvancedLogConfig *config); diff --git a/core/include_internal/ten_utils/log/log.h b/core/include_internal/ten_utils/log/log.h index bdc7fc5989..4955672e13 100644 --- a/core/include_internal/ten_utils/log/log.h +++ b/core/include_internal/ten_utils/log/log.h @@ -43,13 +43,15 @@ TEN_UTILS_PRIVATE_API const char *filename(const char *path, size_t path_len, TEN_UTILS_API void ten_log_log(ten_log_t *self, TEN_LOG_LEVEL level, const char *func_name, const char *file_name, size_t line_no, const char *msg, - const char *category, ten_value_t *fields); + const char *category, ten_value_t *fields, + const ten_log_loc_info_t *loc_info); TEN_UTILS_API void ten_log_log_with_size( ten_log_t *self, TEN_LOG_LEVEL level, const char *func_name, size_t func_name_len, const char *file_name, size_t file_name_len, size_t line_no, const char *msg, size_t msg_len, const char *category, - size_t category_len, ten_value_t *fields); + size_t category_len, ten_value_t *fields, + const ten_log_loc_info_t *loc_info); TEN_UTILS_API void ten_log_global_init(bool enable_advanced_log); diff --git a/core/src/ten_manager/Cargo.lock b/core/src/ten_manager/Cargo.lock index 52954e1a29..b015398694 100644 --- a/core/src/ten_manager/Cargo.lock +++ b/core/src/ten_manager/Cargo.lock @@ -112,7 +112,7 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rand 0.9.2", + "rand", "sha1", "smallvec", "tokio", @@ -1402,12 +1402,6 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" -[[package]] -name = "glob" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" - [[package]] name = "globset" version = "0.4.18" @@ -1433,7 +1427,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.12.1", + "indexmap", "slab", "tokio", "tokio-util", @@ -1452,7 +1446,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.4.0", - "indexmap 2.12.1", + "indexmap", "slab", "tokio", "tokio-util", @@ -1475,12 +1469,6 @@ dependencies = [ "thiserror 2.0.17", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.14.5" @@ -1797,16 +1785,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2" -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.12.1" @@ -1851,7 +1829,7 @@ dependencies = [ "crossbeam-utils", "dashmap", "env_logger", - "indexmap 2.12.1", + "indexmap", "itoa", "log", "num-format", @@ -1949,7 +1927,7 @@ checksum = "01dbdbd07b076e8403abac68ce7744d93e2ecd953bbc44bf77bf00e1e81172bc" dependencies = [ "foldhash", "hifijson", - "indexmap 2.12.1", + "indexmap", "jaq-core", "jaq-std", "serde_json", @@ -2463,9 +2441,9 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "opentelemetry" -version = "0.29.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e87237e2775f74896f9ad219d26a2081751187eb7c9f5c58dde20a23b95d16c" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" dependencies = [ "futures-core", "futures-sink", @@ -2475,27 +2453,37 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry-appender-tracing" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef6a1ac5ca3accf562b8c306fa8483c85f4390f768185ab775f242f7fe8fdcc2" +dependencies = [ + "opentelemetry", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "opentelemetry-http" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46d7ab32b827b5b495bd90fa95a6cb65ccc293555dcc3199ae2937d2d237c8ed" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" dependencies = [ "async-trait", "bytes", "http 1.4.0", "opentelemetry", "reqwest", - "tracing", ] [[package]] name = "opentelemetry-otlp" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d899720fe06916ccba71c01d04ecd77312734e2de3467fd30d9d580c8ce85656" +checksum = "7a2366db2dca4d2ad033cad11e6ee42844fd727007af5ad04a1730f4cb8163bf" dependencies = [ - "futures-core", "http 1.4.0", "opentelemetry", "opentelemetry-http", @@ -2511,9 +2499,9 @@ dependencies = [ [[package]] name = "opentelemetry-prometheus" -version = "0.29.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "098a71a4430bb712be6130ed777335d2e5b19bc8566de5f2edddfce906def6ab" +checksum = "14095eb06b569eb5d538fa4555969f7e8a410ed7910c903bfd295f9e1a50d7ea" dependencies = [ "once_cell", "opentelemetry", @@ -2524,50 +2512,49 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c40da242381435e18570d5b9d50aca2a4f4f4d8e146231adb4e7768023309b3" +checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ "opentelemetry", "opentelemetry_sdk", "prost", "tonic", + "tonic-prost", ] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b29a9f89f1a954936d5aa92f19b2feec3c8f3971d3e96206640db7f9706ae3" +checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" [[package]] name = "opentelemetry-stdout" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7e27d446dabd68610ef0b77d07b102ecde827a4596ea9c01a4d3811e945b286" +checksum = "bc8887887e169414f637b18751487cce4e095be787d23fad13c454e2fb1b3811" dependencies = [ "chrono", - "futures-util", "opentelemetry", "opentelemetry_sdk", ] [[package]] name = "opentelemetry_sdk" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afdefb21d1d47394abc1ba6c57363ab141be19e27cc70d0e422b7f303e4d290b" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" dependencies = [ "futures-channel", "futures-executor", "futures-util", - "glob", "opentelemetry", "percent-encoding", - "rand 0.9.2", - "serde_json", + "rand", "thiserror 2.0.17", - "tracing", + "tokio", + "tokio-stream", ] [[package]] @@ -2769,9 +2756,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.5" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" dependencies = [ "bytes", "prost-derive", @@ -2779,9 +2766,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.5" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", "itertools", @@ -2848,7 +2835,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand", "ring", "rustc-hash", "rustls", @@ -2889,35 +2876,14 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", + "rand_chacha", + "rand_core", ] [[package]] @@ -2927,16 +2893,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.16", + "rand_core", ] [[package]] @@ -3124,7 +3081,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", - "tower 0.5.2", + "tower", "tower-http", "tower-service", "url", @@ -3348,7 +3305,7 @@ version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ - "indexmap 2.12.1", + "indexmap", "itoa", "memchr", "ryu", @@ -3759,6 +3716,7 @@ dependencies = [ "libc", "once_cell", "opentelemetry", + "opentelemetry-appender-tracing", "opentelemetry-otlp", "opentelemetry-prometheus", "opentelemetry-semantic-conventions", @@ -3776,6 +3734,7 @@ dependencies = [ "tokio", "tracing", "tracing-appender", + "tracing-opentelemetry", "tracing-subscriber", "url", "uuid", @@ -3971,9 +3930,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.12.3" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" dependencies = [ "async-trait", "base64", @@ -3986,33 +3945,24 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", + "sync_wrapper", "tokio", "tokio-stream", - "tower 0.4.13", + "tower", "tower-layer", "tower-service", "tracing", ] [[package]] -name = "tower" -version = "0.4.13" +name = "tonic-prost" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand 0.8.5", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", + "bytes", + "prost", + "tonic", ] [[package]] @@ -4023,11 +3973,15 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", + "indexmap", "pin-project-lite", + "slab", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -4043,7 +3997,7 @@ dependencies = [ "http-body", "iri-string", "pin-project-lite", - "tower 0.5.2", + "tower", "tower-layer", "tower-service", ] @@ -4138,6 +4092,25 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6e5658463dd88089aba75c7791e1d3120633b1bfde22478b28f625a9bb1b8e" +dependencies = [ + "js-sys", + "opentelemetry", + "opentelemetry_sdk", + "rustversion", + "smallvec", + "thiserror 2.0.17", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-serde" version = "0.2.0" @@ -4187,7 +4160,7 @@ dependencies = [ "http 1.4.0", "httparse", "log", - "rand 0.9.2", + "rand", "sha1", "thiserror 2.0.17", "utf-8", @@ -4914,7 +4887,7 @@ dependencies = [ "arbitrary", "crc32fast", "flate2", - "indexmap 2.12.1", + "indexmap", "memchr", "time", "zopfli", diff --git a/core/src/ten_runtime/common/loc.c b/core/src/ten_runtime/common/loc.c index 38257b70f4..b0aed85334 100644 --- a/core/src/ten_runtime/common/loc.c +++ b/core/src/ten_runtime/common/loc.c @@ -519,6 +519,35 @@ void ten_loc_set_extension_name(ten_loc_t *self, const char *extension_name) { strlen(extension_name)); } +void ten_log_loc_info_init_from_loc(ten_log_loc_info_t *info, ten_loc_t *loc) { + TEN_ASSERT(info, "Invalid argument."); + TEN_ASSERT(loc && ten_loc_check_integrity(loc), "Invalid argument."); + + if (loc->has_app_uri && !ten_string_is_empty(&loc->app_uri)) { + info->app_uri = ten_string_get_raw_str(&loc->app_uri); + info->app_uri_len = ten_string_len(&loc->app_uri); + } else { + info->app_uri = NULL; + info->app_uri_len = 0; + } + + if (loc->has_graph_id && !ten_string_is_empty(&loc->graph_id)) { + info->graph_id = ten_string_get_raw_str(&loc->graph_id); + info->graph_id_len = ten_string_len(&loc->graph_id); + } else { + info->graph_id = NULL; + info->graph_id_len = 0; + } + + if (loc->has_extension_name && !ten_string_is_empty(&loc->extension_name)) { + info->extension_name = ten_string_get_raw_str(&loc->extension_name); + info->extension_name_len = ten_string_len(&loc->extension_name); + } else { + info->extension_name = NULL; + info->extension_name_len = 0; + } +} + bool ten_loc_str_check_correct(const char *app_uri, const char *graph_id, const char *extension_name, ten_error_t *err) { if (!app_uri) { diff --git a/core/src/ten_runtime/global/log.c b/core/src/ten_runtime/global/log.c index 36fb861c42..5a8128b238 100644 --- a/core/src/ten_runtime/global/log.c +++ b/core/src/ten_runtime/global/log.c @@ -31,7 +31,8 @@ void ten_log_rust_log_func(ten_log_t *self, TEN_LOG_LEVEL level, const char *func_name, size_t func_name_len, const char *file_name, size_t file_name_len, size_t line_no, const char *msg, size_t msg_len, - ten_value_t *fields) { + ten_value_t *fields, + const ten_log_loc_info_t *loc_info) { #if defined(TEN_ENABLE_TEN_RUST_APIS) TEN_ASSERT(self, "Invalid argument."); TEN_ASSERT(self->advanced_impl.impl, "Invalid argument."); @@ -42,7 +43,7 @@ void ten_log_rust_log_func(ten_log_t *self, TEN_LOG_LEVEL level, ten_get_pid_tid(&pid, &tid); ten_rust_log(self->advanced_impl.config, category, category_len, pid, tid, level, func_name, func_name_len, file_name, file_name_len, - line_no, msg, msg_len); + line_no, loc_info, msg, msg_len); #endif } diff --git a/core/src/ten_runtime/ten_env/internal/attached_loc.c b/core/src/ten_runtime/ten_env/internal/attached_loc.c index c76397679f..f9cf164037 100644 --- a/core/src/ten_runtime/ten_env/internal/attached_loc.c +++ b/core/src/ten_runtime/ten_env/internal/attached_loc.c @@ -6,12 +6,14 @@ // #include "include_internal/ten_runtime/addon/addon_host.h" #include "include_internal/ten_runtime/app/app.h" +#include "include_internal/ten_runtime/common/loc.h" #include "include_internal/ten_runtime/engine/engine.h" #include "include_internal/ten_runtime/extension/extension.h" +#include "include_internal/ten_runtime/extension/extension_info/extension_info.h" #include "include_internal/ten_runtime/extension_group/extension_group.h" -#include "include_internal/ten_runtime/extension_group/on_xxx.h" #include "include_internal/ten_runtime/ten_env/ten_env.h" #include "ten_runtime/ten_env/ten_env.h" +#include "ten_utils/macro/check.h" const char *ten_env_get_attached_instance_name(ten_env_t *self, bool check_thread) { @@ -45,3 +47,80 @@ const char *ten_env_get_attached_instance_name(ten_env_t *self, return NULL; } } + +void ten_env_get_attached_target_loc(ten_env_t *self, ten_loc_t *loc, + bool check_thread) { + TEN_ASSERT(self && ten_env_check_integrity(self, check_thread), + "Invalid argument."); + TEN_ASSERT(loc && ten_loc_check_integrity(loc), "Invalid argument."); + + // Clear the location first. + ten_loc_clear(loc); + + switch (self->attach_to) { + case TEN_ENV_ATTACH_TO_EXTENSION: { + ten_extension_t *extension = ten_env_get_attached_extension(self); + TEN_ASSERT(extension && ten_extension_check_integrity(extension, true), + "Invalid extension."); + + // Get location from extension_info. + if (extension->extension_info) { + ten_loc_copy(loc, &extension->extension_info->loc); + } else { + // Fallback: Only set extension name if extension_info is not available. + const char *extension_name = ten_extension_get_name(extension, true); + if (extension_name) { + ten_loc_set_extension_name(loc, extension_name); + } + } + break; + } + + case TEN_ENV_ATTACH_TO_ENGINE: { + ten_engine_t *engine = ten_env_get_attached_engine(self); + TEN_ASSERT(engine && ten_engine_check_integrity(engine, true), + "Invalid engine."); + + // Get app_uri from app if available. + if (engine->app) { + const char *app_uri = ten_app_get_uri(engine->app); + if (app_uri && strlen(app_uri) > 0) { + ten_loc_set_app_uri(loc, app_uri); + } + } + + // Get graph_id from engine. + if (!ten_string_is_empty(&engine->graph_id)) { + ten_loc_set_graph_id(loc, ten_string_get_raw_str(&engine->graph_id)); + } + break; + } + + case TEN_ENV_ATTACH_TO_APP: { + ten_app_t *app = ten_env_get_attached_app(self); + // TEN_NOLINTNEXTLINE(thread-check) + // thread-check: The app uri remains unchanged during the lifecycle of app, + // allowing safe cross-thread access. + TEN_ASSERT(app && ten_app_check_integrity(app, false), "Invalid app."); + + // Get app_uri. + const char *app_uri = ten_app_get_uri(app); + if (app_uri && strlen(app_uri) > 0) { + ten_loc_set_app_uri(loc, app_uri); + } + break; + } + + case TEN_ENV_ATTACH_TO_EXTENSION_GROUP: + case TEN_ENV_ATTACH_TO_ADDON: + case TEN_ENV_ATTACH_TO_ADDON_LOADER: + // These types don't have a well-defined location in the traditional sense + // (app_uri, graph_id, extension_name), so we just clear the location. + // The location remains empty after ten_loc_clear(). + break; + + default: + TEN_ASSERT(0, "Handle more types: %d", self->attach_to); + break; + } +} diff --git a/core/src/ten_runtime/ten_env/internal/log.c b/core/src/ten_runtime/ten_env/internal/log.c index e89f856280..f5417cd37f 100644 --- a/core/src/ten_runtime/ten_env/internal/log.c +++ b/core/src/ten_runtime/ten_env/internal/log.c @@ -6,6 +6,7 @@ // #include "ten_runtime/ten_env/internal/log.h" +#include "include_internal/ten_runtime/common/loc.h" #include "include_internal/ten_runtime/ten_env/ten_env.h" #include "include_internal/ten_utils/log/log.h" #include "ten_runtime/ten_env/ten_env.h" @@ -23,15 +24,28 @@ static void ten_env_log_internal(ten_env_t *self, TEN_LOG_LEVEL level, TEN_ASSERT(self && ten_env_check_integrity(self, check_thread), "Should not happen."); + // Get location information from attached target. + ten_loc_t loc; + ten_loc_init_empty(&loc); + ten_env_get_attached_target_loc(self, &loc, check_thread); + + // Convert ten_loc_t to ten_log_loc_info_t. + ten_log_loc_info_t loc_info; + ten_log_loc_info_init_from_loc(&loc_info, &loc); + ten_string_t final_msg; ten_string_init_formatted( &final_msg, "[%s] %s", ten_env_get_attached_instance_name(self, check_thread), msg); - ten_log_log(&ten_global_log, level, func_name, file_name, line_no, - ten_string_get_raw_str(&final_msg), category, fields); + ten_log_log_with_size( + &ten_global_log, level, func_name, func_name ? strlen(func_name) : 0, + file_name, file_name ? strlen(file_name) : 0, line_no, + ten_string_get_raw_str(&final_msg), ten_string_len(&final_msg), category, + category ? strlen(category) : 0, fields, &loc_info); ten_string_deinit(&final_msg); + ten_loc_deinit(&loc); } // TODO(Wei): This function is currently specifically designed for the addon @@ -69,6 +83,15 @@ static void ten_env_log_with_size_formatted_internal( TEN_ASSERT(self && ten_env_check_integrity(self, check_thread), "Should not happen."); + // Get location information from attached target. + ten_loc_t loc; + ten_loc_init_empty(&loc); + ten_env_get_attached_target_loc(self, &loc, check_thread); + + // Convert ten_loc_t to ten_log_loc_info_t. + ten_log_loc_info_t loc_info; + ten_log_loc_info_init_from_loc(&loc_info, &loc); + ten_string_t final_msg; ten_string_init_formatted( &final_msg, "[%s] ", @@ -79,9 +102,10 @@ static void ten_env_log_with_size_formatted_internal( ten_log_log_with_size( &ten_global_log, level, func_name, func_name_len, file_name, file_name_len, line_no, ten_string_get_raw_str(&final_msg), - ten_string_len(&final_msg), category, category_len, fields); + ten_string_len(&final_msg), category, category_len, fields, &loc_info); ten_string_deinit(&final_msg); + ten_loc_deinit(&loc); } // TODO(Wei): This function is currently specifically designed for the addon diff --git a/core/src/ten_rust/Cargo.lock b/core/src/ten_rust/Cargo.lock index 9fb3440e2e..aae92fb108 100644 --- a/core/src/ten_rust/Cargo.lock +++ b/core/src/ten_rust/Cargo.lock @@ -49,7 +49,7 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rand 0.9.2", + "rand", "sha1", "smallvec", "tokio", @@ -877,7 +877,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.12.1", + "indexmap", "slab", "tokio", "tokio-util", @@ -896,19 +896,13 @@ dependencies = [ "futures-core", "futures-sink", "http 1.4.0", - "indexmap 2.12.1", + "indexmap", "slab", "tokio", "tokio-util", "tracing", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.16.1" @@ -1179,16 +1173,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2" -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.12.1" @@ -1196,7 +1180,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown", ] [[package]] @@ -1553,9 +1537,9 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "opentelemetry" -version = "0.29.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e87237e2775f74896f9ad219d26a2081751187eb7c9f5c58dde20a23b95d16c" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" dependencies = [ "futures-core", "futures-sink", @@ -1565,27 +1549,37 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry-appender-tracing" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef6a1ac5ca3accf562b8c306fa8483c85f4390f768185ab775f242f7fe8fdcc2" +dependencies = [ + "opentelemetry", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "opentelemetry-http" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46d7ab32b827b5b495bd90fa95a6cb65ccc293555dcc3199ae2937d2d237c8ed" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" dependencies = [ "async-trait", "bytes", "http 1.4.0", "opentelemetry", "reqwest", - "tracing", ] [[package]] name = "opentelemetry-otlp" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d899720fe06916ccba71c01d04ecd77312734e2de3467fd30d9d580c8ce85656" +checksum = "7a2366db2dca4d2ad033cad11e6ee42844fd727007af5ad04a1730f4cb8163bf" dependencies = [ - "futures-core", "http 1.4.0", "opentelemetry", "opentelemetry-http", @@ -1601,9 +1595,9 @@ dependencies = [ [[package]] name = "opentelemetry-prometheus" -version = "0.29.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "098a71a4430bb712be6130ed777335d2e5b19bc8566de5f2edddfce906def6ab" +checksum = "14095eb06b569eb5d538fa4555969f7e8a410ed7910c903bfd295f9e1a50d7ea" dependencies = [ "once_cell", "opentelemetry", @@ -1614,50 +1608,49 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c40da242381435e18570d5b9d50aca2a4f4f4d8e146231adb4e7768023309b3" +checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ "opentelemetry", "opentelemetry_sdk", "prost", "tonic", + "tonic-prost", ] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b29a9f89f1a954936d5aa92f19b2feec3c8f3971d3e96206640db7f9706ae3" +checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" [[package]] name = "opentelemetry-stdout" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7e27d446dabd68610ef0b77d07b102ecde827a4596ea9c01a4d3811e945b286" +checksum = "bc8887887e169414f637b18751487cce4e095be787d23fad13c454e2fb1b3811" dependencies = [ "chrono", - "futures-util", "opentelemetry", "opentelemetry_sdk", ] [[package]] name = "opentelemetry_sdk" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afdefb21d1d47394abc1ba6c57363ab141be19e27cc70d0e422b7f303e4d290b" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" dependencies = [ "futures-channel", "futures-executor", "futures-util", - "glob", "opentelemetry", "percent-encoding", - "rand 0.9.2", - "serde_json", + "rand", "thiserror 2.0.17", - "tracing", + "tokio", + "tokio-stream", ] [[package]] @@ -1836,9 +1829,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.5" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" dependencies = [ "bytes", "prost-derive", @@ -1846,9 +1839,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.5" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", "itertools 0.14.0", @@ -1906,7 +1899,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand", "ring", "rustc-hash", "rustls", @@ -1947,35 +1940,14 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", + "rand_chacha", + "rand_core", ] [[package]] @@ -1985,16 +1957,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.16", + "rand_core", ] [[package]] @@ -2114,7 +2077,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", - "tower 0.5.2", + "tower", "tower-http", "tower-service", "url", @@ -2286,7 +2249,7 @@ version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ - "indexmap 2.12.1", + "indexmap", "itoa", "memchr", "ryu", @@ -2489,6 +2452,7 @@ dependencies = [ "libc", "once_cell", "opentelemetry", + "opentelemetry-appender-tracing", "opentelemetry-otlp", "opentelemetry-prometheus", "opentelemetry-semantic-conventions", @@ -2506,6 +2470,7 @@ dependencies = [ "tokio", "tracing", "tracing-appender", + "tracing-opentelemetry", "tracing-subscriber", "url", "uuid", @@ -2680,9 +2645,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.12.3" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203" dependencies = [ "async-trait", "base64", @@ -2695,33 +2660,24 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", + "sync_wrapper", "tokio", "tokio-stream", - "tower 0.4.13", + "tower", "tower-layer", "tower-service", "tracing", ] [[package]] -name = "tower" -version = "0.4.13" +name = "tonic-prost" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand 0.8.5", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", + "bytes", + "prost", + "tonic", ] [[package]] @@ -2732,11 +2688,15 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", + "indexmap", "pin-project-lite", + "slab", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -2752,7 +2712,7 @@ dependencies = [ "http-body", "iri-string", "pin-project-lite", - "tower 0.5.2", + "tower", "tower-layer", "tower-service", ] @@ -2825,6 +2785,25 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6e5658463dd88089aba75c7791e1d3120633b1bfde22478b28f625a9bb1b8e" +dependencies = [ + "js-sys", + "opentelemetry", + "opentelemetry_sdk", + "rustversion", + "smallvec", + "thiserror 2.0.17", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-serde" version = "0.2.0" diff --git a/core/src/ten_rust/Cargo.toml b/core/src/ten_rust/Cargo.toml index c6a729b813..10abaa2afc 100644 --- a/core/src/ten_rust/Cargo.toml +++ b/core/src/ten_rust/Cargo.toml @@ -34,19 +34,22 @@ jsonschema = { version = "0.32", default-features = false } libc = { version = "0.2" } once_cell = "1.19.0" # OpenTelemetry for modern observability (use compatible versions) -opentelemetry = "0.29" -opentelemetry_sdk = "0.29" -opentelemetry-prometheus = "0.29" # Prometheus exporter (for /metrics endpoint) -opentelemetry-stdout = { version = "0.29", features = [ +opentelemetry = "0.31.0" +opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio", "logs"] } +opentelemetry-prometheus = "0.31.0" # Prometheus exporter (for /metrics endpoint) +opentelemetry-stdout = { version = "0.31.0", features = [ "metrics", ] } # Console exporter (for debugging) -opentelemetry-otlp = { version = "0.29", features = [ +opentelemetry-otlp = { version = "0.31.0", features = [ "metrics", + "logs", "grpc-tonic", "http-proto", "reqwest-blocking-client", ] } # OTLP exporter (for pushing to collectors) -opentelemetry-semantic-conventions = "0.29" +opentelemetry-semantic-conventions = "0.31.0" +opentelemetry-appender-tracing = "0.31.0" +tracing-opentelemetry = "0.32.0" prometheus = "0.14" # Prometheus client library for Registry and metrics export regex = { version = "1.11" } semver = { version = "1.0", features = ["serde"] } diff --git a/core/src/ten_rust/src/json_schema/data/property.schema.json b/core/src/ten_rust/src/json_schema/data/property.schema.json index 4d74f6c9dd..cf7bef6c08 100644 --- a/core/src/ten_rust/src/json_schema/data/property.schema.json +++ b/core/src/ten_rust/src/json_schema/data/property.schema.json @@ -880,6 +880,10 @@ "path": { "type": "string", "pattern": "^/.*$" + }, + "service_name": { + "type": "string", + "minLength": 1 } } } @@ -918,6 +922,10 @@ } }, "additionalProperties": false + }, + "service_name": { + "type": "string", + "minLength": 1 } } } @@ -1306,7 +1314,8 @@ "type": "string", "enum": [ "console", - "file" + "file", + "otlp" ] }, "config": { @@ -1372,6 +1381,51 @@ } } } + }, + { + "if": { + "properties": { + "type": { + "const": "otlp" + } + } + }, + "then": { + "properties": { + "config": { + "additionalProperties": false, + "required": [ + "endpoint" + ], + "properties": { + "endpoint": { + "type": "string", + "minLength": 1 + }, + "protocol": { + "type": "string", + "enum": [ + "grpc", + "http" + ] + }, + "headers": { + "type": "object", + "patternProperties": { + "^[A-Za-z0-9_-]+$": { + "type": "string" + } + }, + "additionalProperties": false + }, + "service_name": { + "type": "string", + "minLength": 1 + } + } + } + } + } } ] }, diff --git a/core/src/ten_rust/src/log/bindings.rs b/core/src/ten_rust/src/log/bindings.rs index 73c6bb8b0f..8843b0c95a 100644 --- a/core/src/ten_rust/src/log/bindings.rs +++ b/core/src/ten_rust/src/log/bindings.rs @@ -4,6 +4,53 @@ // Licensed under the Apache License, Version 2.0, with certain conditions. // Refer to the "LICENSE" file in the root directory for more information. // +//! FFI bindings for TEN logging system. +//! +//! This module provides Foreign Function Interface (FFI) bindings between C and +//! Rust for the TEN Framework's logging functionality. +//! +//! # Critical Safety Requirements +//! +//! ## Structure Layout Synchronization +//! +//! The [`TenLogLocInfo`] struct in this module MUST be kept in sync with the C +//! struct `ten_log_loc_info_t` defined in `core/include/ten_utils/log/log.h`. +//! +//! ### Verification Methods +//! +//! 1. **Compile-time checks**: The code includes `const` assertions that +//! verify: +//! - Total struct size matches expected layout +//! - Struct alignment matches pointer alignment +//! - These checks will cause compilation to FAIL if layout is incorrect +//! +//! 2. **Runtime tests**: Run the following command to verify memory layout: +//! ```bash cargo test test_ten_log_loc_info_layout -- --nocapture ``` This +//! test verifies: +//! - Total struct size (48 bytes on 64-bit, 24 bytes on 32-bit) +//! - Struct alignment (8 bytes on 64-bit, 4 bytes on 32-bit) +//! - Individual field offsets match expected C layout +//! +//! Note: The actual test implementation is in +//! `tests/test_case/log/mod.rs::test_ten_log_loc_info_layout` +//! +//! ### Maintenance Workflow +//! +//! **When modifying `TenLogLocInfo` (Rust side):** +//! 1. Update the C definition in `core/include/ten_utils/log/log.h` +//! 2. Run `cargo test test_ten_log_loc_info_layout` to verify +//! 3. Run full C/C++ compilation to ensure compatibility +//! +//! **When modifying `ten_log_loc_info_t` (C side):** +//! 1. Update the Rust definition in this file +//! 2. Run `cargo test test_ten_log_loc_info_layout` to verify +//! 3. Run full C/C++ compilation to ensure compatibility +//! +//! ### What Happens If They Diverge? +//! +//! - **At compile time**: Const assertions will fail, preventing compilation +//! - **At runtime**: Undefined behavior, memory corruption, crashes +//! - **In tests**: Layout verification test will fail with detailed diagnostics use std::{ ffi::{CStr, CString}, os::raw::c_char, @@ -11,6 +58,96 @@ use std::{ use crate::log::{ten_configure_log, ten_log_reopen_all, AdvancedLogConfig}; +/// FFI-safe structure to pass location information from C to Rust. +/// +/// # CRITICAL: Keep in sync with C definition +/// +/// This struct MUST exactly match the C struct `ten_log_loc_info_t` defined in: +/// `core/include/ten_utils/log/log.h` +/// +/// ## C Definition (for reference): +/// ```c +/// typedef struct ten_log_loc_info_t { +/// const char *app_uri; +/// size_t app_uri_len; +/// const char *graph_id; +/// size_t graph_id_len; +/// const char *extension_name; +/// size_t extension_name_len; +/// } ten_log_loc_info_t; +/// ``` +/// +/// ## Memory Layout Requirements: +/// - Field order must match exactly +/// - Field types must have same size and alignment as C counterparts +/// - `#[repr(C)]` ensures C-compatible memory layout +/// +/// ## Safety: +/// - Modifying this struct requires updating the C definition +/// - Modifying the C definition requires updating this struct +/// - Static assertions below verify size and alignment at compile time +/// +/// ## Verification: +/// Run `cargo test test_ten_log_loc_info_layout` to verify layout +/// compatibility. +#[repr(C)] +pub struct TenLogLocInfo { + pub app_uri: *const c_char, + pub app_uri_len: usize, + pub graph_id: *const c_char, + pub graph_id_len: usize, + pub extension_name: *const c_char, + pub extension_name_len: usize, +} + +// Compile-time assertions to ensure struct layout matches C definition. +// These will cause compilation to fail if the struct layout changes +// unexpectedly. +#[allow(clippy::no_effect)] +const _: () = { + // Expected size: 6 fields * pointer/usize size + // On 64-bit: 6 * 8 = 48 bytes + // On 32-bit: 6 * 4 = 24 bytes + const EXPECTED_SIZE: usize = core::mem::size_of::<*const c_char>() * 6; + const ACTUAL_SIZE: usize = core::mem::size_of::(); + + // This will cause a compile error if sizes don't match + const SIZE_MATCHES: bool = EXPECTED_SIZE == ACTUAL_SIZE; + [(); 1][!SIZE_MATCHES as usize]; + + // Verify alignment matches pointer alignment + const EXPECTED_ALIGN: usize = core::mem::align_of::<*const c_char>(); + const ACTUAL_ALIGN: usize = core::mem::align_of::(); + + const ALIGN_MATCHES: bool = EXPECTED_ALIGN == ACTUAL_ALIGN; + [(); 1][!ALIGN_MATCHES as usize]; +}; + +impl TenLogLocInfo { + /// Helper to safely extract strings from the C struct. + pub fn to_strings(&self) -> (&str, &str, &str) { + let app_uri = if self.app_uri_len > 0 && !self.app_uri.is_null() { + unsafe { CStr::from_ptr(self.app_uri).to_str().unwrap_or("") } + } else { + "" + }; + + let graph_id = if self.graph_id_len > 0 && !self.graph_id.is_null() { + unsafe { CStr::from_ptr(self.graph_id).to_str().unwrap_or("") } + } else { + "" + }; + + let extension_name = if self.extension_name_len > 0 && !self.extension_name.is_null() { + unsafe { CStr::from_ptr(self.extension_name).to_str().unwrap_or("") } + } else { + "" + }; + + (app_uri, graph_id, extension_name) + } +} + /// Configure the log. /// /// # Parameter @@ -147,6 +284,7 @@ pub extern "C" fn ten_rust_log( file_name: *const c_char, file_name_len: usize, line_no: u32, + loc_info: *const TenLogLocInfo, msg: *const c_char, msg_len: usize, ) { @@ -165,6 +303,10 @@ pub extern "C" fn ten_rust_log( let log_level = crate::log::LogLevel::from(level as u8); + // Parse location info. + let (app_uri, graph_id, extension_name) = + if !loc_info.is_null() { unsafe { (*loc_info).to_strings() } } else { ("", "", "") }; + let func_name_str = match unsafe { CStr::from_ptr(func_name) }.to_str() { Ok(s) => s, Err(_) => return, @@ -198,6 +340,9 @@ pub extern "C" fn ten_rust_log( func_name_str, file_name_str, line_no, + app_uri, + graph_id, + extension_name, msg_str, ); } diff --git a/core/src/ten_rust/src/log/formatter/json.rs b/core/src/ten_rust/src/log/formatter/json.rs index 78a351b875..f028f3a77f 100644 --- a/core/src/ten_rust/src/log/formatter/json.rs +++ b/core/src/ten_rust/src/log/formatter/json.rs @@ -36,31 +36,31 @@ struct FieldVisitor { impl Visit for FieldVisitor { fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) { match field.name() { - "pid" => { + "ten_pid" => { if let Ok(pid) = format!("{value:?}").parse::() { self.pid = Some(pid); } } - "tid" => { + "ten_tid" => { if let Ok(tid) = format!("{value:?}").parse::() { self.tid = Some(tid); } } - "func_name" => { + "ten_func_name" => { self.func_name = Some(format!("{value:?}").trim_matches('"').to_string()); } - "file_name" => { + "ten_file_name" => { self.file_name = Some(format!("{value:?}").trim_matches('"').to_string()); } - "line_no" => { + "ten_line_no" => { if let Ok(line) = format!("{value:?}").parse::() { self.line_no = Some(line); } } - "category" => { + "ten_category" => { self.category = Some(format!("{value:?}").trim_matches('"').to_string()); } - "message" => { + "ten_message" => { if !self.message.is_empty() { self.message.push(' '); } @@ -80,16 +80,16 @@ impl Visit for FieldVisitor { fn record_str(&mut self, field: &tracing::field::Field, value: &str) { match field.name() { - "func_name" => { + "ten_func_name" => { self.func_name = Some(value.to_string()); } - "file_name" => { + "ten_file_name" => { self.file_name = Some(value.to_string()); } - "category" => { + "ten_category" => { self.category = Some(value.to_string()); } - "message" => { + "ten_message" => { if !self.message.is_empty() { self.message.push(' '); } @@ -105,17 +105,17 @@ impl Visit for FieldVisitor { } fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { - if field.name() == "line_no" { + if field.name() == "ten_line_no" { self.line_no = Some(value as u32); } } fn record_i64(&mut self, field: &tracing::field::Field, value: i64) { match field.name() { - "pid" => { + "ten_pid" => { self.pid = Some(value); } - "tid" => { + "ten_tid" => { self.tid = Some(value); } _ => {} diff --git a/core/src/ten_rust/src/log/formatter/plain.rs b/core/src/ten_rust/src/log/formatter/plain.rs index 43be72b011..70393d1d2e 100644 --- a/core/src/ten_rust/src/log/formatter/plain.rs +++ b/core/src/ten_rust/src/log/formatter/plain.rs @@ -37,31 +37,31 @@ struct FieldVisitor { impl Visit for FieldVisitor { fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn fmt::Debug) { match field.name() { - "pid" => { + "ten_pid" => { if let Ok(pid) = format!("{value:?}").parse::() { self.pid = Some(pid); } } - "tid" => { + "ten_tid" => { if let Ok(tid) = format!("{value:?}").parse::() { self.tid = Some(tid); } } - "func_name" => { + "ten_func_name" => { self.func_name = Some(format!("{value:?}").trim_matches('"').to_string()); } - "file_name" => { + "ten_file_name" => { self.file_name = Some(format!("{value:?}").trim_matches('"').to_string()); } - "line_no" => { + "ten_line_no" => { if let Ok(line) = format!("{value:?}").parse::() { self.line_no = Some(line); } } - "category" => { + "ten_category" => { self.category = Some(format!("{value:?}").trim_matches('"').to_string()); } - "message" => { + "ten_message" => { if !self.message.is_empty() { self.message.push(' '); } @@ -81,16 +81,16 @@ impl Visit for FieldVisitor { fn record_str(&mut self, field: &tracing::field::Field, value: &str) { match field.name() { - "func_name" => { + "ten_func_name" => { self.func_name = Some(value.to_string()); } - "file_name" => { + "ten_file_name" => { self.file_name = Some(value.to_string()); } - "category" => { + "ten_category" => { self.category = Some(value.to_string()); } - "message" => { + "ten_message" => { if !self.message.is_empty() { self.message.push(' '); } @@ -106,17 +106,17 @@ impl Visit for FieldVisitor { } fn record_u64(&mut self, field: &tracing::field::Field, value: u64) { - if field.name() == "line_no" { + if field.name() == "ten_line_no" { self.line_no = Some(value as u32); } } fn record_i64(&mut self, field: &tracing::field::Field, value: i64) { match field.name() { - "pid" => { + "ten_pid" => { self.pid = Some(value); } - "tid" => { + "ten_tid" => { self.tid = Some(value); } _ => {} diff --git a/core/src/ten_rust/src/log/mod.rs b/core/src/ten_rust/src/log/mod.rs index 86e7989a9e..dea8c063f5 100644 --- a/core/src/ten_rust/src/log/mod.rs +++ b/core/src/ten_rust/src/log/mod.rs @@ -10,6 +10,7 @@ pub mod dynamic_filter; pub mod encryption; pub mod file_appender; pub mod formatter; +pub mod otel; pub mod reloadable; use std::{fmt, io}; @@ -136,12 +137,38 @@ pub struct FileEmitterConfig { pub encryption: Option, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum OtlpProtocol { + Grpc, + Http, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct OtlpEmitterConfig { + pub endpoint: String, + + #[serde(default = "default_otlp_protocol")] + pub protocol: OtlpProtocol, + + #[serde(default)] + pub headers: std::collections::HashMap, + + #[serde(skip_serializing_if = "Option::is_none")] + pub service_name: Option, +} + +fn default_otlp_protocol() -> OtlpProtocol { + OtlpProtocol::Grpc +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "type", content = "config")] #[serde(rename_all = "lowercase")] pub enum AdvancedLogEmitter { Console(ConsoleEmitterConfig), File(FileEmitterConfig), + Otlp(OtlpEmitterConfig), } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -326,6 +353,15 @@ fn create_layer_with_dynamic_filter(handler: &AdvancedLogHandler) -> LayerWithGu guard: Some(Box::new(composite_guard)), } } + AdvancedLogEmitter::Otlp(otlp_config) => { + eprintln!("[DEBUG] Creating OTLP layer for endpoint: {}", otlp_config.endpoint); + let (layer, guard) = otel::create_otlp_layer(otlp_config); + eprintln!("[DEBUG] OTLP layer created successfully"); + LayerWithGuard { + layer, + guard: Some(Box::new(guard)), + } + } }; // Wrap the base layer with our dynamic category filter @@ -440,6 +476,9 @@ pub fn ten_log( func_name: &str, file_name: &str, line_no: u32, + app_uri: &str, + graph_id: &str, + extension_name: &str, msg: &str, ) { let tracing_level = level.to_tracing_level(); @@ -451,60 +490,75 @@ pub fn ten_log( match tracing_level { tracing::Level::TRACE => { tracing::trace!( - category = category, - pid = pid, - tid = tid, - func_name = func_name, - file_name = filename, - line_no = line_no, + ten_category = category, + ten_app_uri = app_uri, + ten_graph_id = graph_id, + ten_extension_name = extension_name, + ten_pid = pid, + ten_tid = tid, + ten_func_name = func_name, + ten_file_name = filename, + ten_line_no = line_no, "{}", msg ) } tracing::Level::DEBUG => { tracing::debug!( - category = category, - pid = pid, - tid = tid, - func_name = func_name, - file_name = filename, - line_no = line_no, + ten_category = category, + ten_app_uri = app_uri, + ten_graph_id = graph_id, + ten_extension_name = extension_name, + ten_pid = pid, + ten_tid = tid, + ten_func_name = func_name, + ten_file_name = filename, + ten_line_no = line_no, "{}", msg ) } tracing::Level::INFO => { tracing::info!( - category = category, - pid = pid, - tid = tid, - func_name = func_name, - file_name = filename, - line_no = line_no, + ten_category = category, + ten_app_uri = app_uri, + ten_graph_id = graph_id, + ten_extension_name = extension_name, + ten_pid = pid, + ten_tid = tid, + ten_func_name = func_name, + ten_file_name = filename, + ten_line_no = line_no, "{}", msg ) } tracing::Level::WARN => { tracing::warn!( - category = category, - pid = pid, - tid = tid, - func_name = func_name, - file_name = filename, - line_no = line_no, + ten_category = category, + ten_app_uri = app_uri, + ten_graph_id = graph_id, + ten_extension_name = extension_name, + ten_pid = pid, + ten_tid = tid, + ten_func_name = func_name, + ten_file_name = filename, + ten_line_no = line_no, "{}", msg ) } tracing::Level::ERROR => { tracing::error!( - category = category, - pid = pid, - tid = tid, - func_name = func_name, - file_name = filename, - line_no = line_no, + ten_category = category, + ten_app_uri = app_uri, + ten_graph_id = graph_id, + ten_extension_name = extension_name, + ten_pid = pid, + ten_tid = tid, + ten_func_name = func_name, + ten_file_name = filename, + ten_line_no = line_no, "{}", msg ) diff --git a/core/src/ten_rust/src/log/otel.rs b/core/src/ten_rust/src/log/otel.rs new file mode 100644 index 0000000000..bd06270a4d --- /dev/null +++ b/core/src/ten_rust/src/log/otel.rs @@ -0,0 +1,156 @@ +// +// Copyright © 2025 Agora +// This file is part of TEN Framework, an open source project. +// Licensed under the Apache License, Version 2.0, with certain conditions. +// Refer to the "LICENSE" file in the root directory for more information. +// +use opentelemetry::KeyValue; +use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::{logs::SdkLoggerProvider, Resource}; +use tracing_subscriber::{Layer, Registry}; + +use super::{OtlpEmitterConfig, OtlpProtocol}; + +/// Guard for OTLP telemetry resources +/// +/// This guard holds the logger provider and runtime thread handle. +/// When dropped, it will shutdown the logger provider (flushing all buffered +/// logs) +pub struct OtlpTelemetryGuard { + provider: Option, + // Keep the runtime thread alive + _runtime_handle: Option>, +} + +impl Drop for OtlpTelemetryGuard { + fn drop(&mut self) { + // Skip shutdown if we're panicking or if TLS might be destroyed + if std::thread::panicking() { + return; + } + + if let Some(provider) = self.provider.take() { + // Use eprintln! instead of tracing macros to avoid TLS access during shutdown + eprintln!("[OTLP] Shutting down OpenTelemetry logger provider..."); + + // Attempt to shutdown, but don't panic if it fails due to TLS issues + // This can happen during process exit when TLS is being destroyed + let shutdown_result = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| provider.shutdown())); + + match shutdown_result { + Ok(Ok(())) => { + eprintln!("[OTLP] Logger provider shut down successfully"); + } + Ok(Err(e)) => { + eprintln!("[OTLP] Failed to shutdown logger provider: {:?}", e); + } + Err(_) => { + eprintln!( + "[OTLP] Logger provider shutdown panicked (likely due to TLS destruction \ + during process exit)" + ); + } + } + } + } +} + +pub fn create_otlp_layer( + config: &OtlpEmitterConfig, +) -> (Box + Send + Sync>, OtlpTelemetryGuard) { + let service_name = config.service_name.clone().unwrap_or_else(|| "ten-framework".to_string()); + let endpoint = config.endpoint.clone(); + let protocol = config.protocol.clone(); + let _headers = config.headers.clone(); // TODO: Add header support when API available + + eprintln!( + "[OTLP] Initializing OTLP log layer: endpoint={}, service_name={}, protocol={:?}", + endpoint, service_name, protocol + ); + + // Channel to receive the LoggerProvider + let (tx, rx) = std::sync::mpsc::channel(); + let endpoint_for_thread = endpoint.clone(); + + let handle = std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().expect("Failed to create Tokio runtime"); + rt.block_on(async { + let resource = Resource::builder() + .with_service_name(service_name) + .with_attributes(vec![KeyValue::new("service.namespace", "ten-framework")]) + .build(); + + // Setup OTLP log exporter based on protocol + eprintln!("[OTLP] Creating log exporter for endpoint: {}", endpoint_for_thread); + let exporter_result = match protocol { + OtlpProtocol::Grpc => { + eprintln!("[OTLP] Using gRPC protocol"); + opentelemetry_otlp::LogExporter::builder() + .with_tonic() + .with_endpoint(&endpoint_for_thread) + .build() + } + OtlpProtocol::Http => { + eprintln!("[OTLP] Using HTTP protocol"); + opentelemetry_otlp::LogExporter::builder() + .with_http() + .with_endpoint(&endpoint_for_thread) + .build() + } + }; + + let exporter = match exporter_result { + Ok(exp) => { + eprintln!("[OTLP] Log exporter created successfully"); + exp + } + Err(e) => { + eprintln!("[OTLP] FAILED to create OTLP log exporter!"); + eprintln!("[OTLP] Error: {:?}", e); + eprintln!("[OTLP] Endpoint: {}", endpoint_for_thread); + eprintln!("[OTLP] Protocol: {:?}", protocol); + eprintln!("[OTLP] Logs will NOT be exported to OTLP endpoint!"); + return; + } + }; + + // Setup Logger Provider + eprintln!("[OTLP] Creating logger provider with batch exporter..."); + let provider = SdkLoggerProvider::builder() + .with_batch_exporter(exporter) + .with_resource(resource) + .build(); + + eprintln!("[OTLP] Logger provider created successfully"); + + // Send provider back to main thread + if tx.send(provider).is_err() { + eprintln!("[OTLP] Failed to send logger provider to main thread"); + } + + // Keep the runtime alive + std::future::pending::<()>().await; + }); + }); + + // Wait for logger provider + let provider = rx.recv().expect("Failed to receive logger provider"); + + // Create the OpenTelemetry tracing bridge layer + let layer = OpenTelemetryTracingBridge::new(&provider); + + eprintln!("[OTLP] OTLP log layer created and ready"); + eprintln!("[OTLP] Note: If you see 'BatchLogProcessor.ExportError', check:"); + eprintln!("[OTLP] 1. Is the OTLP collector running at {}?", endpoint); + eprintln!("[OTLP] 2. Is the endpoint URL correct?"); + eprintln!("[OTLP] 3. Check network connectivity and firewall rules"); + + let guard = OtlpTelemetryGuard { + provider: Some(provider), + _runtime_handle: Some(handle), + }; + + (Box::new(layer), guard) +} diff --git a/core/src/ten_rust/src/service_hub/mod.rs b/core/src/ten_rust/src/service_hub/mod.rs index f38359ddf0..288c3a5254 100644 --- a/core/src/ten_rust/src/service_hub/mod.rs +++ b/core/src/ten_rust/src/service_hub/mod.rs @@ -524,7 +524,9 @@ pub unsafe extern "C" fn ten_service_hub_create( let exporter_type = telemetry_config .as_ref() .map(telemetry::ExporterType::from_config) - .unwrap_or(telemetry::ExporterType::Prometheus); + .unwrap_or(telemetry::ExporterType::Prometheus { + service_name: None, + }); // Initialize metrics exporter. let metrics_exporter = telemetry::MetricsExporter::new(exporter_type); diff --git a/core/src/ten_rust/src/service_hub/telemetry/config.rs b/core/src/ten_rust/src/service_hub/telemetry/config.rs index 9732977acb..88483eb056 100644 --- a/core/src/ten_rust/src/service_hub/telemetry/config.rs +++ b/core/src/ten_rust/src/service_hub/telemetry/config.rs @@ -125,6 +125,10 @@ pub struct PrometheusConfig { /// Metrics endpoint path (default: "/metrics") #[serde(default = "default_prometheus_path")] pub path: String, + + /// Service name (optional, defaults to application name) + #[serde(skip_serializing_if = "Option::is_none")] + pub service_name: Option, } /// OTLP exporter configuration (Push mode) @@ -140,6 +144,10 @@ pub struct OtlpConfig { /// HTTP headers for authentication #[serde(default)] pub headers: HashMap, + + /// Service name (optional, defaults to application name) + #[serde(skip_serializing_if = "Option::is_none")] + pub service_name: Option, } /// OTLP protocol type @@ -185,6 +193,7 @@ impl Default for PrometheusConfig { Self { endpoint: default_prometheus_endpoint(), path: default_prometheus_path(), + service_name: None, } } } diff --git a/core/src/ten_rust/src/service_hub/telemetry/exporter.rs b/core/src/ten_rust/src/service_hub/telemetry/exporter.rs index 0be121fc66..6a362b2833 100644 --- a/core/src/ten_rust/src/service_hub/telemetry/exporter.rs +++ b/core/src/ten_rust/src/service_hub/telemetry/exporter.rs @@ -17,13 +17,14 @@ use super::config::{OtlpProtocol, TelemetryConfig}; #[derive(Debug, Clone)] pub enum ExporterType { /// Prometheus exporter (Pull mode, /metrics endpoint) - Prometheus, + Prometheus { service_name: Option }, /// OTLP exporter (Push mode, to Collector/Langfuse/etc) Otlp { endpoint: String, protocol: OtlpProtocol, headers: std::collections::HashMap, + service_name: Option, }, /// Console exporter (for debugging) @@ -112,7 +113,11 @@ impl ExporterType { match exporter_type { ConfigExporterType::Prometheus => { tracing::info!("Telemetry: Using Prometheus exporter (Pull mode)"); - ExporterType::Prometheus + ExporterType::Prometheus { + service_name: config + .get_prometheus_config() + .and_then(|c| c.service_name.clone()), + } } ConfigExporterType::Otlp => { if let Some(otlp_config) = config.get_otlp_config() { @@ -127,13 +132,16 @@ impl ExporterType { endpoint: otlp_config.endpoint.clone(), protocol: otlp_config.protocol.clone(), headers: otlp_config.headers.clone(), + service_name: otlp_config.service_name.clone(), } } else { tracing::warn!( "Warning: OTLP exporter selected but no config provided, falling back to \ Prometheus" ); - ExporterType::Prometheus + ExporterType::Prometheus { + service_name: None, + } } } ConfigExporterType::Console => { @@ -167,15 +175,26 @@ impl MetricsExporter { } /// Initialize the exporter with given service name - pub fn init(&self, service_name: &str) -> Result<()> { + pub fn init(&self, default_service_name: &str) -> Result<()> { match &self.exporter_type { - ExporterType::Prometheus => self.init_prometheus_exporter(service_name), + ExporterType::Prometheus { + service_name, + } => { + let effective_service_name = + service_name.as_deref().unwrap_or(default_service_name); + self.init_prometheus_exporter(effective_service_name) + } ExporterType::Otlp { endpoint, protocol, headers, - } => self.init_otlp_exporter(service_name, endpoint, protocol, headers), - ExporterType::Console => self.init_console_exporter(service_name), + service_name, + } => { + let effective_service_name = + service_name.as_deref().unwrap_or(default_service_name); + self.init_otlp_exporter(effective_service_name, endpoint, protocol, headers) + } + ExporterType::Console => self.init_console_exporter(default_service_name), } } @@ -374,6 +393,8 @@ impl MetricsExporter { impl Default for MetricsExporter { fn default() -> Self { - Self::new(ExporterType::Prometheus) + Self::new(ExporterType::Prometheus { + service_name: None, + }) } } diff --git a/core/src/ten_rust/tests/test_case/log/mod.rs b/core/src/ten_rust/tests/test_case/log/mod.rs index 2c6c21d618..2e546c0df3 100644 --- a/core/src/ten_rust/tests/test_case/log/mod.rs +++ b/core/src/ten_rust/tests/test_case/log/mod.rs @@ -137,6 +137,9 @@ mod tests { "test_func", "test.rs", 100, + "", + "", + "", "Trace message", ); ten_log( @@ -148,6 +151,9 @@ mod tests { "test_func", "test.rs", 101, + "", + "", + "", "Debug message", ); ten_log( @@ -159,6 +165,9 @@ mod tests { "test_func", "test.rs", 102, + "", + "", + "", "Info message", ); ten_log( @@ -170,6 +179,9 @@ mod tests { "test_func", "test.rs", 103, + "", + "", + "", "Warn message", ); ten_log( @@ -181,6 +193,9 @@ mod tests { "test_func", "test.rs", 104, + "", + "", + "", "Error message", ); @@ -230,6 +245,9 @@ mod tests { "test_plain_colored", "formatter.rs", 50, + "", + "", + "", "Error message in red", ); @@ -242,6 +260,9 @@ mod tests { "test_plain_colored", "formatter.rs", 51, + "", + "", + "", "Warning message in yellow", ); @@ -254,6 +275,9 @@ mod tests { "test_plain_colored", "formatter.rs", 52, + "", + "", + "", "Info message in default color", ); @@ -266,6 +290,9 @@ mod tests { "test_plain_colored", "formatter.rs", 53, + "", + "", + "", "Debug message in blue", ); } @@ -299,6 +326,9 @@ mod tests { "test_plain_no_color", "formatter.rs", 51, + "", + "", + "", "Plain no color message", ); } @@ -331,6 +361,9 @@ mod tests { "test_json", "formatter.rs", 52, + "", + "", + "", "JSON formatted message", ); } @@ -363,6 +396,9 @@ mod tests { "test_json_colored", "formatter.rs", 53, + "", + "", + "", "JSON colored message", ); @@ -375,6 +411,9 @@ mod tests { "test_json_colored", "formatter.rs", 54, + "", + "", + "", "JSON colored message", ); } @@ -407,6 +446,9 @@ mod tests { "test_stdout", "emitter.rs", 60, + "", + "", + "", "Message to stdout", ); } @@ -439,6 +481,9 @@ mod tests { "test_stderr", "emitter.rs", 61, + "", + "", + "", "Warning message to stderr", ); } @@ -474,6 +519,9 @@ mod tests { "test_file_plain", "file_emitter.rs", 70, + "", + "", + "", "Plain message to file", ); ten_log( @@ -485,6 +533,9 @@ mod tests { "test_file_plain", "file_emitter.rs", 71, + "", + "", + "", "Warning message to file", ); @@ -530,6 +581,9 @@ mod tests { "test_file_json", "file_emitter.rs", 80, + "", + "", + "", "JSON message to file", ); @@ -590,6 +644,9 @@ mod tests { "before_fn", "before.rs", 1, + "", + "", + "", "before-1", ); ten_log( @@ -601,6 +658,9 @@ mod tests { "before_fn", "before.rs", 2, + "", + "", + "", "before-2", ); @@ -615,8 +675,34 @@ mod tests { ten_rust::log::ten_log_reopen_all(&mut config, true); // Write more lines after reopen request - ten_log(&config, "test_reopen", 1, 1, LogLevel::Info, "after_fn", "after.rs", 3, "after-1"); - ten_log(&config, "test_reopen", 1, 1, LogLevel::Warn, "after_fn", "after.rs", 4, "after-2"); + ten_log( + &config, + "test_reopen", + 1, + 1, + LogLevel::Info, + "after_fn", + "after.rs", + 3, + "", + "", + "", + "after-1", + ); + ten_log( + &config, + "test_reopen", + 1, + 1, + LogLevel::Warn, + "after_fn", + "after.rs", + 4, + "", + "", + "", + "after-2", + ); // Force flush: disable all handlers to drop worker guard(s) ten_configure_log_reloadable(&AdvancedLogConfig::new(vec![])).unwrap(); @@ -688,6 +774,9 @@ mod tests { "encrypt_test", "encrypt.rs", 1, + "", + "", + "", msg, ); ten_log( @@ -699,6 +788,9 @@ mod tests { "encrypt_test", "encrypt.rs", 1, + "", + "", + "", "My card number is 1234567890", ); ten_log( @@ -710,6 +802,9 @@ mod tests { "encrypt_test", "encrypt.rs", 1, + "", + "", + "", "My phone number is 9876543210", ); @@ -969,6 +1064,9 @@ mod tests { "test_default", "default.rs", 100, + "", + "", + "", "Default config info", ); } @@ -1024,6 +1122,9 @@ mod tests { "test_fn", "test.rs", 1, + "", + "", + "", &format!("log message {counter}"), ); counter += 1; @@ -1112,6 +1213,9 @@ mod tests { "test_fn", "test.rs", 1, + "", + "", + "", &format!("log message {counter}"), ); counter += 1; @@ -1192,6 +1296,9 @@ mod tests { "main", "app.rs", 10, + "", + "", + "", "Application started", ); ten_log( @@ -1203,6 +1310,9 @@ mod tests { "auth", "auth.rs", 25, + "", + "", + "", "User login successful", ); ten_log( @@ -1214,6 +1324,9 @@ mod tests { "db", "database.rs", 50, + "", + "", + "", "Database connection pool almost full", ); ten_log( @@ -1225,6 +1338,9 @@ mod tests { "network", "network.rs", 75, + "", + "", + "", "Network connection timeout", ); @@ -1237,6 +1353,9 @@ mod tests { "parser", "json_parser.rs", 100, + "", + "", + "", "Parse JSON: {\"key\": \"value\"}", ); } @@ -1269,9 +1388,48 @@ mod tests { ten_configure_log_reloadable(&config).unwrap(); // These should all be dropped due to global OFF - ten_log(&config, "any", 1, 1, LogLevel::Info, "f", "f.rs", 1, "global-off-info"); - ten_log(&config, "any", 1, 1, LogLevel::Warn, "f", "f.rs", 2, "global-off-warn"); - ten_log(&config, "any", 1, 1, LogLevel::Error, "f", "f.rs", 3, "global-off-error"); + ten_log( + &config, + "any", + 1, + 1, + LogLevel::Info, + "f", + "f.rs", + 1, + "", + "", + "", + "global-off-info", + ); + ten_log( + &config, + "any", + 1, + 1, + LogLevel::Warn, + "f", + "f.rs", + 2, + "", + "", + "", + "global-off-warn", + ); + ten_log( + &config, + "any", + 1, + 1, + LogLevel::Error, + "f", + "f.rs", + 3, + "", + "", + "", + "global-off-error", + ); // Force flush logs (drop workers) ten_configure_log_reloadable(&AdvancedLogConfig::new(vec![])).unwrap(); @@ -1309,10 +1467,10 @@ mod tests { ten_configure_log_reloadable(&config).unwrap(); // These should all be dropped due to global OFF - ten_log(&config, "auth", 1, 1, LogLevel::Info, "f", "f.rs", 1, "aaa"); - ten_log(&config, "auth", 1, 1, LogLevel::Warn, "f", "f.rs", 2, "bbb"); - ten_log(&config, "database", 1, 1, LogLevel::Error, "f", "f.rs", 3, "ccc"); - ten_log(&config, "", 1, 1, LogLevel::Error, "f", "f.rs", 3, "ddd"); + ten_log(&config, "auth", 1, 1, LogLevel::Info, "f", "f.rs", 1, "", "", "", "aaa"); + ten_log(&config, "auth", 1, 1, LogLevel::Warn, "f", "f.rs", 2, "", "", "", "bbb"); + ten_log(&config, "database", 1, 1, LogLevel::Error, "f", "f.rs", 3, "", "", "", "ccc"); + ten_log(&config, "", 1, 1, LogLevel::Error, "f", "f.rs", 3, "", "", "", "ddd"); // Force flush logs (drop workers) ten_configure_log_reloadable(&AdvancedLogConfig::new(vec![])).unwrap(); @@ -1415,8 +1573,34 @@ mod tests { ten_configure_log_reloadable(&config).unwrap(); // These should be written (effective level is DEBUG) - ten_log(&config, "any", 1, 1, LogLevel::Debug, "f", "f.rs", 1, "off-then-debug-debug"); - ten_log(&config, "any", 1, 1, LogLevel::Info, "f", "f.rs", 2, "off-then-debug-info"); + ten_log( + &config, + "any", + 1, + 1, + LogLevel::Debug, + "f", + "f.rs", + 1, + "", + "", + "", + "off-then-debug-debug", + ); + ten_log( + &config, + "any", + 1, + 1, + LogLevel::Info, + "f", + "f.rs", + 2, + "", + "", + "", + "off-then-debug-info", + ); // Force flush ten_configure_log_reloadable(&AdvancedLogConfig::new(vec![])).unwrap(); @@ -1428,4 +1612,150 @@ mod tests { assert!(content.contains("off-then-debug-debug")); assert!(content.contains("off-then-debug-info")); } + + /// Comprehensive compile-time and runtime verification of TenLogLocInfo + /// layout. + /// + /// This test ensures that the Rust struct `TenLogLocInfo` has the exact + /// same memory layout as the C struct `ten_log_loc_info_t`. + /// + /// ## Checks performed: + /// 1. Total struct size + /// 2. Struct alignment + /// 3. Individual field offsets + /// 4. Field sizes + /// + /// If this test fails, it means the Rust and C structs are out of sync. + #[test] + #[serial] + fn test_ten_log_loc_info_layout() { + use std::{ + mem::{align_of, offset_of, size_of}, + os::raw::c_char, + }; + + use ten_rust::log::bindings::TenLogLocInfo; + + // Verify overall size + // Expected: 6 fields of pointer-sized values + // On 64-bit: 48 bytes (6 * 8) + // On 32-bit: 24 bytes (6 * 4) + let ptr_size = size_of::<*const c_char>(); + let expected_size = ptr_size * 6; + assert_eq!( + size_of::(), + expected_size, + "TenLogLocInfo size mismatch: expected {}, got {}", + expected_size, + size_of::() + ); + + // Verify alignment (should match pointer alignment) + assert_eq!( + align_of::(), + align_of::<*const c_char>(), + "TenLogLocInfo alignment mismatch" + ); + + // Verify field offsets match expected C struct layout + // In C: fields are laid out sequentially without padding (for this struct) + assert_eq!(offset_of!(TenLogLocInfo, app_uri), 0, "app_uri offset should be 0"); + assert_eq!( + offset_of!(TenLogLocInfo, app_uri_len), + ptr_size, + "app_uri_len offset should be {}", + ptr_size + ); + assert_eq!( + offset_of!(TenLogLocInfo, graph_id), + ptr_size * 2, + "graph_id offset should be {}", + ptr_size * 2 + ); + assert_eq!( + offset_of!(TenLogLocInfo, graph_id_len), + ptr_size * 3, + "graph_id_len offset should be {}", + ptr_size * 3 + ); + assert_eq!( + offset_of!(TenLogLocInfo, extension_name), + ptr_size * 4, + "extension_name offset should be {}", + ptr_size * 4 + ); + assert_eq!( + offset_of!(TenLogLocInfo, extension_name_len), + ptr_size * 5, + "extension_name_len offset should be {}", + ptr_size * 5 + ); + + // Verify individual field sizes + assert_eq!(size_of::<*const c_char>(), ptr_size, "Pointer size mismatch"); + assert_eq!( + size_of::(), + ptr_size, + "usize should match pointer size on this platform" + ); + + println!("✓ TenLogLocInfo layout verification passed"); + println!(" Size: {} bytes", size_of::()); + println!(" Alignment: {} bytes", align_of::()); + println!(" Field offsets:"); + println!(" app_uri: {}", offset_of!(TenLogLocInfo, app_uri)); + println!(" app_uri_len: {}", offset_of!(TenLogLocInfo, app_uri_len)); + println!(" graph_id: {}", offset_of!(TenLogLocInfo, graph_id)); + println!(" graph_id_len: {}", offset_of!(TenLogLocInfo, graph_id_len)); + println!(" extension_name: {}", offset_of!(TenLogLocInfo, extension_name)); + println!(" extension_name_len: {}", offset_of!(TenLogLocInfo, extension_name_len)); + } + + /// Test the to_strings method with NULL pointers and empty strings. + #[test] + #[serial] + fn test_ten_log_loc_info_to_strings_null() { + use ten_rust::log::bindings::TenLogLocInfo; + + let loc_info = TenLogLocInfo { + app_uri: std::ptr::null(), + app_uri_len: 0, + graph_id: std::ptr::null(), + graph_id_len: 0, + extension_name: std::ptr::null(), + extension_name_len: 0, + }; + + let (app_uri, graph_id, extension_name) = loc_info.to_strings(); + assert_eq!(app_uri, ""); + assert_eq!(graph_id, ""); + assert_eq!(extension_name, ""); + } + + /// Test the to_strings method with valid C strings. + #[test] + #[serial] + fn test_ten_log_loc_info_to_strings_valid() { + use std::ffi::CString; + + use ten_rust::log::bindings::TenLogLocInfo; + + let app_uri_str = CString::new("msgpack://127.0.0.1:8001/").unwrap(); + let graph_id_str = CString::new("default").unwrap(); + let ext_name_str = CString::new("test_extension").unwrap(); + + let loc_info = TenLogLocInfo { + app_uri: app_uri_str.as_ptr(), + app_uri_len: app_uri_str.as_bytes().len(), + graph_id: graph_id_str.as_ptr(), + graph_id_len: graph_id_str.as_bytes().len(), + extension_name: ext_name_str.as_ptr(), + extension_name_len: ext_name_str.as_bytes().len(), + }; + + let (app_uri, graph_id, extension_name) = loc_info.to_strings(); + assert_eq!(app_uri, "msgpack://127.0.0.1:8001/"); + assert_eq!(graph_id, "default"); + assert_eq!(extension_name, "test_extension"); + } } diff --git a/core/src/ten_rust/tests/test_case/telemetry/console_exporter.rs b/core/src/ten_rust/tests/test_case/telemetry/console_exporter.rs index e3076aa528..c4266886e4 100644 --- a/core/src/ten_rust/tests/test_case/telemetry/console_exporter.rs +++ b/core/src/ten_rust/tests/test_case/telemetry/console_exporter.rs @@ -144,7 +144,7 @@ fn test_multiple_exporters_comparison() { // Test Prometheus exporter println!("📊 Testing Prometheus Exporter:"); - let prom_exporter = MetricsExporter::new(ExporterType::Prometheus); + let prom_exporter = MetricsExporter::new(ExporterType::Prometheus { service_name: None }); assert!(prom_exporter.init("prom-service").is_ok()); assert!(prom_exporter.get_prometheus_registry().is_some()); println!(" ✅ Prometheus exporter works, has registry\n"); diff --git a/core/src/ten_utils/log/log.c b/core/src/ten_utils/log/log.c index 29ae899917..0fcf008ac0 100644 --- a/core/src/ten_utils/log/log.c +++ b/core/src/ten_utils/log/log.c @@ -153,7 +153,7 @@ static void ten_log_log_from_va_list(ten_log_t *self, TEN_LOG_LEVEL level, ten_string_init_from_va_list(&msg, fmt, ap); ten_log_log(self, level, func_name, file_name, line_no, - ten_string_get_raw_str(&msg), category, fields); + ten_string_get_raw_str(&msg), category, fields, NULL); ten_string_deinit(&msg); } @@ -176,14 +176,15 @@ void ten_log_log_formatted(ten_log_t *self, TEN_LOG_LEVEL level, void ten_log_log(ten_log_t *self, TEN_LOG_LEVEL level, const char *func_name, const char *file_name, size_t line_no, const char *msg, - const char *category, ten_value_t *fields) { + const char *category, ten_value_t *fields, + const ten_log_loc_info_t *loc_info) { TEN_ASSERT(self, "Invalid argument."); TEN_ASSERT(ten_log_check_integrity(self), "Invalid argument."); ten_log_log_with_size( self, level, func_name, func_name ? strlen(func_name) : 0, file_name, file_name ? strlen(file_name) : 0, line_no, msg, msg ? strlen(msg) : 0, - category, category ? strlen(category) : 0, fields); + category, category ? strlen(category) : 0, fields, loc_info); } void ten_log_log_with_size(ten_log_t *self, TEN_LOG_LEVEL level, @@ -191,7 +192,8 @@ void ten_log_log_with_size(ten_log_t *self, TEN_LOG_LEVEL level, const char *file_name, size_t file_name_len, size_t line_no, const char *msg, size_t msg_len, const char *category, size_t category_len, - ten_value_t *fields) { + ten_value_t *fields, + const ten_log_loc_info_t *loc_info) { TEN_ASSERT(self, "Invalid argument."); TEN_ASSERT(ten_log_check_integrity(self), "Invalid argument."); @@ -199,7 +201,7 @@ void ten_log_log_with_size(ten_log_t *self, TEN_LOG_LEVEL level, if (self->advanced_impl.impl) { self->advanced_impl.impl(self, level, category, category_len, func_name, func_name_len, file_name, file_name_len, line_no, - msg, msg_len, fields); + msg, msg_len, fields, loc_info); } return; diff --git a/core/ten_gn b/core/ten_gn index ff25cb78ce..0f293df9fc 160000 --- a/core/ten_gn +++ b/core/ten_gn @@ -1 +1 @@ -Subproject commit ff25cb78ce95abab92f9b3930d89d83bd02fd0ee +Subproject commit 0f293df9fcca0e2fd08568fb1044281dd894b0ca diff --git a/tests/ten_runtime/integration/cpp/long_running/long_running_app/property.json b/tests/ten_runtime/integration/cpp/long_running/long_running_app/property.json index e7506a0afe..89f9f1bd8a 100644 --- a/tests/ten_runtime/integration/cpp/long_running/long_running_app/property.json +++ b/tests/ten_runtime/integration/cpp/long_running/long_running_app/property.json @@ -32,6 +32,24 @@ "stream": "stdout" } } + }, + { + "matchers": [ + { + "level": "info" + } + ], + "formatter": { + "type": "json", + "colored": false + }, + "emitter": { + "type": "otlp", + "config": { + "endpoint": "http://localhost:4317", + "service_name": "my-custom-service-name" + } + } } ] }, diff --git a/tests/ten_runtime/integration/cpp/long_running/long_running_app/ten_packages/extension/extension_1/src/main.cc b/tests/ten_runtime/integration/cpp/long_running/long_running_app/ten_packages/extension/extension_1/src/main.cc index d1c41a1779..b553aca8c1 100644 --- a/tests/ten_runtime/integration/cpp/long_running/long_running_app/ten_packages/extension/extension_1/src/main.cc +++ b/tests/ten_runtime/integration/cpp/long_running/long_running_app/ten_packages/extension/extension_1/src/main.cc @@ -24,6 +24,7 @@ class test_extension : public ten::extension_t { void on_cmd(ten::ten_env_t &ten_env, std::unique_ptr cmd) override { if (cmd->get_name() == "test_cmd_from_2") { + TEN_ENV_LOG_INFO(ten_env, "test_cmd_from_2 received"); auto cmd_result = ten::cmd_result_t::create(TEN_STATUS_CODE_OK, *cmd); ten_env.return_result(std::move(cmd_result)); @@ -32,6 +33,7 @@ class test_extension : public ten::extension_t { if (cnt < 100) { cnt++; + TEN_ENV_LOG_INFO(ten_env, "test_cmd_from_1 sent"); auto test_cmd = ten::cmd_t::create("test_cmd_from_1"); ten_env.send_cmd(std::move(test_cmd)); } else { diff --git a/tests/ten_runtime/integration/cpp/long_running/long_running_app/ten_packages/extension/extension_2/src/main.cc b/tests/ten_runtime/integration/cpp/long_running/long_running_app/ten_packages/extension/extension_2/src/main.cc index 347e5533b3..aa38b35fdd 100644 --- a/tests/ten_runtime/integration/cpp/long_running/long_running_app/ten_packages/extension/extension_2/src/main.cc +++ b/tests/ten_runtime/integration/cpp/long_running/long_running_app/ten_packages/extension/extension_2/src/main.cc @@ -16,11 +16,13 @@ class test_extension : public ten::extension_t { void on_cmd(ten::ten_env_t &ten_env, std::unique_ptr cmd) override { if (cmd->get_name() == "test_cmd_from_1") { + TEN_ENV_LOG_INFO(ten_env, "test_cmd_from_1 received"); auto cmd_result = ten::cmd_result_t::create(TEN_STATUS_CODE_OK, *cmd); ten_env.return_result(std::move(cmd_result)); ten_random_sleep_range_ms(1000, 2000); + TEN_ENV_LOG_INFO(ten_env, "test_cmd_from_2 sent"); auto test_cmd = ten::cmd_t::create("test_cmd_from_2"); ten_env.send_cmd(std::move(test_cmd)); } diff --git a/tools/grafana-monitoring/README.md b/tools/grafana-monitoring/README.md index 8a1dedd164..594faf7438 100644 --- a/tools/grafana-monitoring/README.md +++ b/tools/grafana-monitoring/README.md @@ -1,6 +1,13 @@ # TEN Framework - Grafana Performance Monitoring -This directory provides a complete monitoring solution for TEN Framework applications using Prometheus and Grafana. +This directory provides a complete monitoring solution for TEN Framework applications using Prometheus, Loki, and Grafana. + +## 🎯 Features + +- ✅ **Metrics Monitoring**: Real-time performance metrics with Prometheus +- ✅ **Log Aggregation**: Centralized log collection with Loki +- ✅ **Unified Visualization**: Grafana dashboards for metrics and logs +- ✅ **OTLP Support**: OpenTelemetry Protocol for cloud-native observability ## 🎯 Deployment Modes @@ -9,7 +16,7 @@ We provide **three deployment configurations** to suit different needs: | Mode | Use Case | Deploy Command | Config Files | |------|----------|----------------|--------------| | **Pull Mode** | Development, Simple Deployment | `docker-compose -f docker-compose.pull.yml up -d` | `configs/pull/` | -| **Push Mode** | Production, Cloud Native | `docker-compose -f docker-compose.push.yml up -d` | `configs/push/` | +| **Push Mode** | Production, Cloud Native, **Logs + Metrics** | `docker-compose -f docker-compose.push.yml up -d` | `configs/push/` | | **Hybrid Mode** | Testing Both Modes | `docker-compose -f docker-compose.hybrid.yml up -d` | `configs/hybrid/` | --- @@ -20,6 +27,9 @@ We provide **three deployment configurations** to suit different needs: **Architecture:** Application exposes metrics endpoint → Prometheus scrapes periodically +**Features:** +- ✅ Metrics only (no logs) + **Pros:** - ✅ Simplest setup (only 2 components) @@ -30,6 +40,7 @@ We provide **three deployment configurations** to suit different needs: - ❌ Requires port exposure - ❌ Cannot capture shutdown metrics (on_stop, on_deinit) +- ❌ No log aggregation **Best For:** Development, testing, long-running services @@ -37,21 +48,28 @@ We provide **three deployment configurations** to suit different needs: ### 2. Push Mode (OTLP Exporter) ⭐ Recommended for Production -**Architecture:** Application pushes metrics → OTEL Collector → Prometheus +**Architecture:** Application pushes metrics + logs → OTEL Collector → Prometheus + Loki → Grafana + +**Features:** +- ✅ Metrics collection (OpenTelemetry → Prometheus) +- ✅ Log aggregation (OpenTelemetry → Loki) +- ✅ Unified visualization in Grafana **Pros:** - ✅ No port exposure needed (more secure) - ✅ Captures full lifecycle metrics (on_stop, on_deinit) +- ✅ **Centralized log collection and visualization** - ✅ Cloud-native architecture - ✅ Supports complex data routing and multi-backend export +- ✅ Correlate metrics and logs in same dashboard **Cons:** -- ❌ Requires OTEL Collector deployment +- ❌ Requires OTEL Collector and Loki deployment - ❌ More complex configuration -**Best For:** Production, cloud-native deployments, Kubernetes, short-lived processes +**Best For:** Production, cloud-native deployments, Kubernetes, short-lived processes, **applications needing log visibility** --- @@ -78,65 +96,42 @@ We provide **three deployment configurations** to suit different needs: | Feature | Pull Mode | Push Mode | Hybrid Mode | |---------|-----------|-----------|-------------| | **Setup Complexity** | ⭐ Simple | ⭐⭐ Medium | ⭐⭐ Medium | -| **Components** | 2 | 3 | 3 | +| **Components** | 2 | 4 (+ Loki) | 4 (+ Loki) | | **Port Exposure** | Required | Not Required | Optional | | **Full Lifecycle** | ❌ | ✅ | Configurable | +| **Log Aggregation** | ❌ | ✅ | ✅ | | **Cloud Native** | Basic | ✅ | ✅ | | **Best For** | Dev/Test | Production | Testing/Migration | --- -## 🚀 Quick Start +## 🚀 Quick Start with Logs -### Pull Mode (Simplest) +### Push Mode (Production - Metrics + Logs) ```bash -# 1. Start monitoring stack +# 1. Start monitoring stack (Prometheus + Loki + Grafana) cd /path/to/grafana-monitoring -docker-compose -f docker-compose.pull.yml up -d +docker-compose -f docker-compose.push.yml up -d # 2. Configure your TEN application (property.json) { "ten": { - "services": { - "telemetry": { - "enabled": true, - "metrics": { - "enabled": true, - "exporter": { - "type": "prometheus", + "log": { + "handlers": [ + { + "matchers": [{"level": "debug"}], + "formatter": {"type": "json"}, + "emitter": { + "type": "otlp", "config": { - "endpoint": "0.0.0.0:49484", - "path": "/metrics" + "endpoint": "http://localhost:4317", + "service_name": "my-ten-service" } } } - } - } - } -} - -# 3. Start your TEN application -./your_ten_application - -# 4. Access Grafana -# URL: http://localhost:3001 -# Username: admin -# Password: admin -``` - ---- - -### Push Mode (Production) - -```bash -# 1. Start monitoring stack -cd /path/to/grafana-monitoring -docker-compose -f docker-compose.push.yml up -d - -# 2. Configure your TEN application (property.json) -{ - "ten": { + ] + }, "services": { "telemetry": { "enabled": true, @@ -146,7 +141,8 @@ docker-compose -f docker-compose.push.yml up -d "type": "otlp", "config": { "endpoint": "http://localhost:4317", - "protocol": "grpc" + "protocol": "grpc", + "service_name": "my-ten-service" } } } @@ -158,14 +154,20 @@ docker-compose -f docker-compose.push.yml up -d # 3. Start your TEN application ./your_ten_application -# 4. Verify +# 4. Verify metrics docker logs -f ten_otel_collector_push curl http://localhost:8889/metrics | grep ten_ -# 5. Access Grafana +# 5. Query logs +curl -G "http://localhost:3100/loki/api/v1/query" \ + --data-urlencode 'query={service_name="my-ten-service"}' + +# 6. Access Grafana # URL: http://localhost:3001 # Username: admin # Password: admin +# - Explore → Select "Loki" data source to query logs +# - Dashboards → TEN Framework metrics dashboard ``` --- @@ -188,6 +190,62 @@ docker-compose -f docker-compose.hybrid.yml up -d --- +## 📝 Log Visualization (Push Mode Only) + +### Accessing Logs in Grafana + +1. **Open Grafana**: http://localhost:3001 (admin/admin) + +2. **Navigate to Explore**: + - Click the compass icon (🧭) in the left sidebar + - Select "Loki" from the data source dropdown + +3. **Query Logs with LogQL**: + +```logql +# View all logs from your service +{service_name="my-ten-service"} + +# Filter by log level +{service_name="my-ten-service", level="error"} + +# Search for specific text +{service_name="my-ten-service"} |= "database connection" + +# Regex search +{service_name="my-ten-service"} |~ "error|warn" + +# Rate of log lines +rate({service_name="my-ten-service"}[5m]) + +# Count errors by level +sum by (level) (rate({service_name="my-ten-service"}[5m])) +``` + +4. **Create Log Dashboard**: + - Add new panel to existing dashboard + - Select "Loki" as data source + - Choose visualization: Logs, Time series, or Table + - Save dashboard + +### Correlating Metrics and Logs + +In Grafana, you can create dashboards that show both metrics and logs: + +``` ++------------------+------------------+ +| Error Rate | Request Rate | ← Metrics (Prometheus) +| (Prometheus) | (Prometheus) | ++------------------+------------------+ +| Error Logs | ← Logs (Loki) +| (Loki) | ++------------------------------------+ +``` + +**See detailed guide**: `LOGS_VISUALIZATION.md` + +--- + ## 📊 Monitored Metrics ### 1. Extension Lifecycle Duration @@ -430,9 +488,10 @@ docker-compose -f docker-compose.pull.yml restart ### Push Mode Ports -- **4317:** OTLP gRPC receiver -- **4318:** OTLP HTTP receiver +- **4317:** OTLP gRPC receiver (metrics + logs) +- **4318:** OTLP HTTP receiver (metrics + logs) - **8889:** OTEL Collector Prometheus exporter +- **3100:** Loki HTTP API - **9091:** Prometheus UI - **3001:** Grafana UI diff --git a/tools/grafana-monitoring/configs/push/loki-config.yml b/tools/grafana-monitoring/configs/push/loki-config.yml new file mode 100644 index 0000000000..7b2f839f06 --- /dev/null +++ b/tools/grafana-monitoring/configs/push/loki-config.yml @@ -0,0 +1,50 @@ +# Loki Configuration for TEN Framework Log Storage + +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +# Query settings +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +# Compactor for cleanup and retention +compactor: + working_directory: /loki/compactor + compaction_interval: 10m + retention_enabled: true + retention_delete_delay: 2h + retention_delete_worker_count: 150 + delete_request_store: filesystem + +# Limits for retention +limits_config: + retention_period: 168h diff --git a/tools/grafana-monitoring/configs/push/otel-collector-config.yml b/tools/grafana-monitoring/configs/push/otel-collector-config.yml index 166aba5644..997438ae37 100644 --- a/tools/grafana-monitoring/configs/push/otel-collector-config.yml +++ b/tools/grafana-monitoring/configs/push/otel-collector-config.yml @@ -1,5 +1,5 @@ # OpenTelemetry Collector Configuration - Push Mode -# Receives OTLP metrics and exports to Prometheus +# Receives OTLP metrics and logs, exports to Prometheus and Loki receivers: otlp: @@ -25,7 +25,7 @@ processors: action: upsert exporters: - # Export to Prometheus (pull mode) + # Export metrics to Prometheus (pull mode) prometheus: endpoint: "0.0.0.0:8889" # namespace: ten # Removed to keep metric names consistent with Pull Mode @@ -34,6 +34,12 @@ exporters: resource_to_telemetry_conversion: enabled: true + # Export logs to Loki via OTLP HTTP + otlphttp/loki: + endpoint: http://loki:3100/otlp + tls: + insecure: true + # For debugging: output to logs debug: verbosity: normal @@ -42,11 +48,18 @@ exporters: service: pipelines: + # Metrics pipeline metrics: receivers: [otlp] processors: [batch, resource] exporters: [prometheus, debug] + # Logs pipeline + logs: + receivers: [otlp] + processors: [batch, resource] + exporters: [otlphttp/loki, debug] + telemetry: logs: level: info diff --git a/tools/grafana-monitoring/docker-compose.push.yml b/tools/grafana-monitoring/docker-compose.push.yml index 1a8ef9a7bb..2372684cda 100644 --- a/tools/grafana-monitoring/docker-compose.push.yml +++ b/tools/grafana-monitoring/docker-compose.push.yml @@ -1,24 +1,43 @@ # TEN Framework - Push Mode Monitoring Configuration # Use Case: Application uses OTLP exporter, cloud-native deployment # +# Components: +# - OpenTelemetry Collector: Receives OTLP metrics and logs +# - Prometheus: Stores metrics (pulled from OTel Collector) +# - Loki: Stores logs (receives from OTel Collector via OTLP) +# - Grafana: Visualizes metrics and logs +# # Usage: # docker-compose -f docker-compose.push.yml up -d # +# Access: +# - Grafana UI: http://localhost:3001 (admin/admin) +# - Prometheus UI: http://localhost:9091 +# # Application configuration example (property.json): -# { -# "type": "otlp", -# "config": { -# "endpoint": "http://localhost:4317", -# "protocol": "grpc" +# Metrics: +# { +# "type": "otlp", +# "config": { +# "endpoint": "http://localhost:4317", +# "protocol": "grpc" +# } +# } +# Logs: +# { +# "type": "otlp", +# "config": { +# "endpoint": "http://localhost:4317", +# "service_name": "my-service" +# } # } -# } networks: monitoring: driver: bridge services: - # OpenTelemetry Collector - Receives OTLP metrics + # OpenTelemetry Collector - Receives OTLP metrics and logs otel-collector: image: otel/opentelemetry-collector-contrib:latest container_name: ten_otel_collector_push @@ -35,6 +54,21 @@ services: - "host.docker.internal:host-gateway" restart: unless-stopped + # Loki - Log aggregation system + loki: + image: grafana/loki:latest + container_name: ten_loki_push + ports: + - "3100:3100" # Loki HTTP + volumes: + - loki_data:/loki + - ./configs/push/loki-config.yml:/etc/loki/local-config.yaml + command: ["-config.file=/etc/loki/local-config.yaml"] + networks: + - monitoring + restart: unless-stopped + + # Prometheus - Metrics storage (pulls from OTel Collector) prometheus: image: prom/prometheus:latest container_name: ten_prometheus_push @@ -54,6 +88,7 @@ services: - otel-collector restart: unless-stopped + # Grafana - Visualization (metrics + traces/logs) grafana: image: grafana/grafana:latest container_name: ten_grafana_push @@ -66,12 +101,15 @@ services: - GF_SECURITY_ADMIN_USER=admin - GF_SECURITY_ADMIN_PASSWORD=admin - GF_USERS_ALLOW_SIGN_UP=false + - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor networks: - monitoring depends_on: - prometheus + - loki restart: unless-stopped volumes: prometheus_data: grafana_data: + loki_data: diff --git a/tools/grafana-monitoring/grafana/provisioning/datasources/loki.yml b/tools/grafana-monitoring/grafana/provisioning/datasources/loki.yml new file mode 100644 index 0000000000..efe6be6b97 --- /dev/null +++ b/tools/grafana-monitoring/grafana/provisioning/datasources/loki.yml @@ -0,0 +1,17 @@ +apiVersion: 1 + +datasources: + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + isDefault: false + version: 1 + editable: true + jsonData: + maxLines: 1000 + derivedFields: + - datasourceUid: prometheus + matcherRegex: "traceID=(\\w+)" + name: TraceID + url: "$${__value.raw}"