diff --git a/build-support/build_opentelemetry.sh b/build-support/build_opentelemetry.sh new file mode 100755 index 000000000000..3a486dcb10b9 --- /dev/null +++ b/build-support/build_opentelemetry.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash + +# Copyright (c) YugabyteDB, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations +# under the License. + +# Build OpenTelemetry C++ SDK with static libraries for YugabyteDB + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +YB_SRC_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Configuration +OTEL_VERSION="v1.24.0" +# Default to installing in thirdparty directory (no sudo required) +OTEL_INSTALL_PREFIX="${OTEL_INSTALL_PREFIX:-${YB_SRC_ROOT}/thirdparty/installed/opentelemetry}" +BUILD_DIR="/tmp/opentelemetry-cpp-build" +NUM_JOBS="${NUM_JOBS:-$(sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4)}" + +echo "================================================================" +echo "Building OpenTelemetry C++ SDK ${OTEL_VERSION}" +echo "================================================================" +echo "Install prefix: ${OTEL_INSTALL_PREFIX}" +echo "Build directory: ${BUILD_DIR}" +echo "Parallel jobs: ${NUM_JOBS}" +echo "" + +# Check for required tools +for tool in git cmake make; do + if ! command -v "$tool" &> /dev/null; then + echo "ERROR: $tool is required but not installed" + exit 1 + fi +done + +# Clean up old build directory +if [[ -d "${BUILD_DIR}" ]]; then + echo "Removing old build directory..." + rm -rf "${BUILD_DIR}" +fi + +# Clone OpenTelemetry C++ SDK +echo "Cloning OpenTelemetry C++ SDK..." +git clone --recurse-submodules --depth 1 --branch "${OTEL_VERSION}" \ + https://github.com/open-telemetry/opentelemetry-cpp.git "${BUILD_DIR}" + +cd "${BUILD_DIR}" + +# Create build directory +mkdir -p build +cd build + +echo "" +echo "Configuring OpenTelemetry build..." +echo "" + +# Configure with CMake +# - Static libraries only (BUILD_SHARED_LIBS=OFF) +# - OTLP HTTP exporter (avoids gRPC/protobuf conflicts with YugabyteDB) +# - No examples, tests, or benchmarks +# - Use libc++ to match YugabyteDB's ABI (required for Linux builds) + +# On Linux, use clang with libc++ to match YugabyteDB's ABI +# macOS uses libc++ by default with Apple clang +CMAKE_EXTRA_FLAGS="" +if [[ "$(uname)" != "Darwin" ]]; then + # Find clang in YB's LLVM installation + CLANG_PATH="" + CLANGXX_PATH="" + + # Check /opt/yb-build/llvm for YB's clang installation + for llvm_dir in /opt/yb-build/llvm/yb-llvm-*/; do + if [[ -x "${llvm_dir}bin/clang++" ]]; then + CLANG_PATH="${llvm_dir}bin/clang" + CLANGXX_PATH="${llvm_dir}bin/clang++" + break + fi + done + + # Fallback to system clang if not found + if [[ -z "${CLANGXX_PATH}" ]]; then + if command -v clang++ &> /dev/null; then + CLANG_PATH="clang" + CLANGXX_PATH="clang++" + fi + fi + + if [[ -n "${CLANGXX_PATH}" ]]; then + echo "Using clang: ${CLANGXX_PATH}" + CMAKE_EXTRA_FLAGS="-DCMAKE_C_COMPILER=${CLANG_PATH} -DCMAKE_CXX_COMPILER=${CLANGXX_PATH} -DCMAKE_CXX_FLAGS=-stdlib=libc++ -DCMAKE_EXE_LINKER_FLAGS=-stdlib=libc++ -DCMAKE_SHARED_LINKER_FLAGS=-stdlib=libc++" + else + echo "WARNING: clang++ not found, using default compiler (may cause ABI issues)" + fi +fi + +# shellcheck disable=SC2086 +cmake .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${OTEL_INSTALL_PREFIX}" \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_CXX_STANDARD=17 \ + -DBUILD_SHARED_LIBS=OFF \ + -DWITH_OTLP_GRPC=OFF \ + -DWITH_OTLP_HTTP=OFF \ + -DWITH_HTTP_CLIENT_CURL=OFF \ + -DWITH_PROMETHEUS=OFF \ + -DWITH_ZIPKIN=OFF \ + -DWITH_JAEGER=OFF \ + -DWITH_ELASTICSEARCH=OFF \ + -DWITH_EXAMPLES=OFF \ + -DWITH_LOGS_PREVIEW=OFF \ + -DWITH_METRICS_PREVIEW=OFF \ + -DBUILD_TESTING=OFF \ + -DWITH_BENCHMARK=OFF \ + ${CMAKE_EXTRA_FLAGS} + +echo "" +echo "Building OpenTelemetry (this may take a few minutes)..." +echo "" + +# Build +make -j"${NUM_JOBS}" + +echo "" +echo "Installing OpenTelemetry to ${OTEL_INSTALL_PREFIX}..." +echo "" + +# Create install prefix directory if it doesn't exist +mkdir -p "${OTEL_INSTALL_PREFIX}" + +# Install (no sudo - use a writable prefix or set OTEL_INSTALL_PREFIX) +if [[ -w "${OTEL_INSTALL_PREFIX}" ]]; then + make install +else + echo "ERROR: Install directory ${OTEL_INSTALL_PREFIX} is not writable" + echo "Either:" + echo " 1. Set OTEL_INSTALL_PREFIX to a writable location, or" + echo " 2. Create the directory with appropriate permissions first" + exit 1 +fi + +# Verify installation +echo "" +echo "Verifying installation..." +if [[ -f "${OTEL_INSTALL_PREFIX}/include/opentelemetry/version.h" ]]; then + echo "✓ Headers installed" +else + echo "✗ Headers not found" + exit 1 +fi + +if [[ -f "${OTEL_INSTALL_PREFIX}/lib/libopentelemetry_trace.a" ]]; then + echo "✓ Static libraries installed" +else + echo "✗ Static libraries not found" + exit 1 +fi + +# List installed libraries +echo "" +echo "Installed static libraries:" +ls -lh "${OTEL_INSTALL_PREFIX}/lib/"*.a | awk '{print " " $9 " (" $5 ")"}' + +# Clean up build directory +echo "" +echo "Removing build directory ${BUILD_DIR}..." +rm -rf "${BUILD_DIR}" +echo "Build directory removed" + +echo "" +echo "================================================================" +echo "OpenTelemetry C++ SDK installed successfully!" +echo "================================================================" +echo "" +echo "Installation directory: ${OTEL_INSTALL_PREFIX}" +echo "" +echo "Next steps:" +echo " 1. Rebuild YugabyteDB: ./yb_build.sh release" +echo " 2. CMake will automatically detect and use OpenTelemetry" +echo "" +echo "To uninstall:" +echo " rm -rf ${OTEL_INSTALL_PREFIX}" +echo "" + diff --git a/dev.yml b/dev.yml new file mode 100644 index 000000000000..98d7a4178477 --- /dev/null +++ b/dev.yml @@ -0,0 +1,44 @@ +name: yugabyte-db + +up: + - packages: + - llvm@16 + - cmake + - go: 1.24.6 + - python: 3.12.7 + + - custom: + name: Verify Clang 16 + met?: /opt/homebrew/opt/llvm@16/bin/clang --version | grep -q "clang version 16" + meet: echo "Clang 16 installed via llvm@16" + +# - custom: +# name: Verify CMake >= 3.31 +# met?: cmake --version | grep -E "cmake version (3\.3[1-9]|3\.[4-9][0-9]|[4-9]\.[0-9]+)" +# meet: echo "CMake version check passed" + + # Setup Yugabyte build dir + - custom: + name: Setup build directory + met?: test -d /opt/yb-build && [ "$(stat -f '%Su' /opt/yb-build 2>/dev/null || echo '')" = "$USER" ] + meet: sudo mkdir -p /opt/yb-build && sudo chown "$USER" /opt/yb-build + +env: + # Point to Clang 16 + CC: /opt/homebrew/opt/llvm@16/bin/clang + CXX: /opt/homebrew/opt/llvm@16/bin/clang++ + LDFLAGS: "-L/opt/homebrew/opt/llvm@16/lib" + CPPFLAGS: "-I/opt/homebrew/opt/llvm@16/include" + +commands: + build: + desc: Build Yugabyte + run: ./yb_build.sh release + +# test: +# desc: Run tests +# run: ./build/your-test-binary + +# clean: +# desc: Clean build artifacts +# run: rm -rf build diff --git a/src/postgres/src/backend/access/transam/xact.c b/src/postgres/src/backend/access/transam/xact.c index ea1ddda3f80e..3257649a01e5 100644 --- a/src/postgres/src/backend/access/transam/xact.c +++ b/src/postgres/src/backend/access/transam/xact.c @@ -2450,6 +2450,7 @@ CommitTransaction(void) * Postgres transaction can be aborted at this point without an issue * in case of YBCCommitTransaction failure. */ + YBCOtelCommitStart(); YBCCommitTransaction(); if (increment_pg_txns) YbIncrementPgTxnsCommitted(); @@ -2486,6 +2487,7 @@ CommitTransaction(void) } TRACE_POSTGRESQL_TRANSACTION_COMMIT(MyProc->lxid); + YBCOtelCommitDone(); /* * Let others know about no transaction in progress by me. Note that this @@ -3044,6 +3046,7 @@ AbortTransaction(void) XLogSetAsyncXactLSN(XactLastRecEnd); } + YBCOtelAbortStart(); TRACE_POSTGRESQL_TRANSACTION_ABORT(MyProc->lxid); /* @@ -3095,6 +3098,7 @@ AbortTransaction(void) } YBCAbortTransaction(); + YBCOtelAbortDone(); /* Reset the value of the sticky connection */ s->ybUncommittedStickyObjectCount = 0; diff --git a/src/postgres/src/backend/tcop/postgres.c b/src/postgres/src/backend/tcop/postgres.c index a7c7fc350498..ca7182c6eb25 100644 --- a/src/postgres/src/backend/tcop/postgres.c +++ b/src/postgres/src/backend/tcop/postgres.c @@ -692,6 +692,7 @@ pg_parse_query(const char *query_string) List *raw_parsetree_list; TRACE_POSTGRESQL_QUERY_PARSE_START(query_string); + YBCOtelParseStart(); if (log_parser_stats) ResetUsage(); @@ -720,6 +721,7 @@ pg_parse_query(const char *query_string) * here. */ + YBCOtelParseDone(); TRACE_POSTGRESQL_QUERY_PARSE_DONE(query_string); return raw_parsetree_list; @@ -766,6 +768,7 @@ pg_analyze_and_rewrite_fixedparams(RawStmt *parsetree, List *querytree_list; TRACE_POSTGRESQL_QUERY_REWRITE_START(query_string); + YBCOtelRewriteStart(); /* * (1) Perform parse analysis. @@ -784,6 +787,7 @@ pg_analyze_and_rewrite_fixedparams(RawStmt *parsetree, */ querytree_list = pg_rewrite_query(query); + YBCOtelRewriteDone(); TRACE_POSTGRESQL_QUERY_REWRITE_DONE(query_string); return querytree_list; @@ -805,6 +809,7 @@ pg_analyze_and_rewrite_varparams(RawStmt *parsetree, List *querytree_list; TRACE_POSTGRESQL_QUERY_REWRITE_START(query_string); + YBCOtelRewriteStart(); /* * (1) Perform parse analysis. @@ -837,6 +842,7 @@ pg_analyze_and_rewrite_varparams(RawStmt *parsetree, */ querytree_list = pg_rewrite_query(query); + YBCOtelRewriteDone(); TRACE_POSTGRESQL_QUERY_REWRITE_DONE(query_string); return querytree_list; @@ -859,6 +865,7 @@ pg_analyze_and_rewrite_withcb(RawStmt *parsetree, List *querytree_list; TRACE_POSTGRESQL_QUERY_REWRITE_START(query_string); + YBCOtelRewriteStart(); /* * (1) Perform parse analysis. @@ -877,6 +884,7 @@ pg_analyze_and_rewrite_withcb(RawStmt *parsetree, */ querytree_list = pg_rewrite_query(query); + YBCOtelRewriteDone(); TRACE_POSTGRESQL_QUERY_REWRITE_DONE(query_string); return querytree_list; @@ -994,6 +1002,7 @@ pg_plan_query(Query *querytree, const char *query_string, int cursorOptions, Assert(ActiveSnapshotSet()); TRACE_POSTGRESQL_QUERY_PLAN_START(); + YBCOtelPlanStart(); if (log_planner_stats) ResetUsage(); @@ -1053,6 +1062,7 @@ pg_plan_query(Query *querytree, const char *query_string, int cursorOptions, if (Debug_print_plan) elog_node_display(LOG, "plan", plan, Debug_pretty_print); + YBCOtelPlanDone(); TRACE_POSTGRESQL_QUERY_PLAN_DONE(); return plan; @@ -1127,12 +1137,16 @@ exec_simple_query(const char *query_string) */ debug_query_string = query_string; + /* Parse traceparent from query comment for OTEL tracing */ + YbSetTraceparentFromQuery(query_string); + /* Use YbParseCommandTag to suppress error warnings. */ command_tag = YbParseCommandTag(query_string); redacted_query_string = YbRedactPasswordIfExists(query_string, command_tag); pgstat_report_activity(STATE_RUNNING, redacted_query_string); TRACE_POSTGRESQL_QUERY_START(query_string); + YBCOtelQueryStart(query_string); /* * We use save_log_statement_stats so ShowUsage doesn't report incorrect @@ -1479,6 +1493,7 @@ exec_simple_query(const char *query_string) if (save_log_statement_stats) ShowUsage("QUERY STATISTICS"); + YBCOtelQueryDone(); TRACE_POSTGRESQL_QUERY_DONE(query_string); debug_query_string = NULL; diff --git a/src/postgres/src/backend/tcop/pquery.c b/src/postgres/src/backend/tcop/pquery.c index 686bed2a7ec9..89def9388d2a 100644 --- a/src/postgres/src/backend/tcop/pquery.c +++ b/src/postgres/src/backend/tcop/pquery.c @@ -32,6 +32,7 @@ #include "executor/ybModifyTable.h" #include "optimizer/ybplan.h" #include "pg_yb_utils.h" +#include "yb/yql/pggate/ybc_pggate.h" /* @@ -713,6 +714,7 @@ PortalRun(Portal portal, long count, bool isTopLevel, bool run_once, AssertArg(PortalIsValid(portal)); TRACE_POSTGRESQL_QUERY_EXECUTE_START(); + YBCOtelExecuteStart(); /* Initialize empty completion data */ if (qc) @@ -865,6 +867,7 @@ PortalRun(Portal portal, long count, bool isTopLevel, bool run_once, if (log_executor_stats && portal->strategy != PORTAL_MULTI_QUERY) ShowUsage("EXECUTOR STATISTICS"); + YBCOtelExecuteDone(); TRACE_POSTGRESQL_QUERY_EXECUTE_DONE(); return result; diff --git a/src/postgres/src/backend/utils/misc/pg_yb_utils.c b/src/postgres/src/backend/utils/misc/pg_yb_utils.c index 29b0d284c936..4a87b633f873 100644 --- a/src/postgres/src/backend/utils/misc/pg_yb_utils.c +++ b/src/postgres/src/backend/utils/misc/pg_yb_utils.c @@ -1058,6 +1058,13 @@ YBInitPostgresBackend(const char *program_name, uint64_t *session_id) { HandleYBStatus(YBCInit(program_name, palloc, cstring_to_text_with_len)); + /* + * Initialize OpenTelemetry tracing for this postgres backend process. + * Each backend process needs its own OTEL initialization since they are + * separate processes with separate memory spaces. + */ + YBCInitOtelTracing("postgres-backend"); + /* * Enable "YB mode" for PostgreSQL so that we will initiate a connection * to the YugaByte cluster right away from every backend process. We only @@ -7945,3 +7952,110 @@ YbUseTserverResponseCacheForAuth(uint64_t shared_catalog_version) return false; return true; } + +/* Buffer to store the current traceparent for OTEL tracing */ +#define YB_TRACEPARENT_MAX_LEN 128 +static char yb_current_traceparent[YB_TRACEPARENT_MAX_LEN] = {0}; + +/* + * Parse and store the traceparent from a SQL query comment. + * Expected format: comment with traceparent:00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01 + */ +void +YbSetTraceparentFromQuery(const char *query_string) +{ + const char *p; + const char *traceparent_start; + const char *traceparent_end; + size_t len; + + /* Clear any existing traceparent */ + yb_current_traceparent[0] = '\0'; + + if (!query_string) + return; + + /* Skip leading whitespace */ + p = query_string; + while (*p && (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r')) + p++; + + /* Check if query starts with a block comment */ + if (p[0] != '/' || p[1] != '*') + { + /* No comment - silently return to reduce log noise */ + return; + } + + /* Move past the opening comment delimiter */ + p += 2; + + /* Skip whitespace inside the comment */ + while (*p && (*p == ' ' || *p == '\t')) + p++; + + /* Look for "traceparent:" (case-sensitive) */ + if (strncmp(p, "traceparent:", 12) != 0) + { + /* No traceparent in comment - silently return to reduce log noise */ + return; + } + + /* Move past "traceparent:" */ + p += 12; + + /* The traceparent value starts here */ + traceparent_start = p; + + /* Find the end of the traceparent (either whitespace or closing comment) */ + traceparent_end = traceparent_start; + while (*traceparent_end && *traceparent_end != ' ' && *traceparent_end != '\t' && + *traceparent_end != '\n' && *traceparent_end != '\r' && + *traceparent_end != '*') + traceparent_end++; + + len = traceparent_end - traceparent_start; + + /* Ensure we don't overflow the buffer */ + if (len >= YB_TRACEPARENT_MAX_LEN) + len = YB_TRACEPARENT_MAX_LEN - 1; + + if (len > 0) + { + memcpy(yb_current_traceparent, traceparent_start, len); + yb_current_traceparent[len] = '\0'; + + ereport(LOG, + (errmsg("[OTEL DEBUG] Parsed traceparent from query: '%s'", + yb_current_traceparent))); + } + else + { + ereport(LOG, + (errmsg("[OTEL DEBUG] Empty traceparent value in comment"))); + } +} + +/* + * Get the current traceparent string. + */ +const char * +YbGetCurrentTraceparent(void) +{ + return yb_current_traceparent; +} + +/* + * Clear the current traceparent. + */ +void +YbClearTraceparent(void) +{ + if (yb_current_traceparent[0] != '\0') + { + ereport(LOG, + (errmsg("[OTEL DEBUG] Clearing traceparent: '%s'", + yb_current_traceparent))); + yb_current_traceparent[0] = '\0'; + } +} diff --git a/src/postgres/src/include/pg_yb_utils.h b/src/postgres/src/include/pg_yb_utils.h index 9512b15efbd8..dc5edcfcf4e1 100644 --- a/src/postgres/src/include/pg_yb_utils.h +++ b/src/postgres/src/include/pg_yb_utils.h @@ -351,6 +351,25 @@ extern void YBCRollbackToSubTransaction(SubTransactionId id); */ extern bool YBIsPgLockingEnabled(); +/* + * Parse and store the traceparent from a SQL query comment. + * Expected format: comment with traceparent:00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01 + * This should be called at the start of query execution. + */ +extern void YbSetTraceparentFromQuery(const char *query_string); + +/* + * Get the current traceparent string (may be empty if none was set). + * Returns a pointer to an internal buffer that is valid until the next call to + * YbSetTraceparentFromQuery or YbClearTraceparent. + */ +extern const char *YbGetCurrentTraceparent(void); + +/* + * Clear the current traceparent. + */ +extern void YbClearTraceparent(void); + /* * Get the type ID of a real or virtual attribute (column). * Returns InvalidOid if the attribute number is invalid. diff --git a/src/yb/rpc/outbound_call.cc b/src/yb/rpc/outbound_call.cc index ac55648dbf05..040cb5156890 100644 --- a/src/yb/rpc/outbound_call.cc +++ b/src/yb/rpc/outbound_call.cc @@ -58,6 +58,7 @@ #include "yb/util/logging.h" #include "yb/util/memory/memory.h" #include "yb/util/metrics.h" +#include "yb/util/otel_tracing.h" #include "yb/util/pb_util.h" #include "yb/util/result.h" #include "yb/util/scope_exit.h" @@ -241,6 +242,20 @@ OutboundCall::OutboundCall(const RemoteMethod& remote_method, IncrementCounter(rpc_metrics_->outbound_calls_created); IncrementGauge(rpc_metrics_->outbound_calls_alive); + + // Create RPC span inheriting from current context (e.g., pggate.batch) + if (OtelTracing::HasActiveContext()) { + otel_span_ = OtelTracing::StartSpan( + Format("rpc.client $0", remote_method_.ToString())); + if (otel_span_.IsActive()) { + otel_span_.SetAttribute("rpc.service", remote_method_.service_name()); + otel_span_.SetAttribute("rpc.method", remote_method_.method_name()); + otel_span_.SetAttribute("rpc.call_id", static_cast(call_id_)); + if (controller_->timeout().Initialized()) { + otel_span_.SetAttribute("rpc.timeout_ms", controller_->timeout().ToMilliseconds()); + } + } + } } OutboundCall::~OutboundCall() { @@ -554,6 +569,11 @@ void OutboundCall::SetResponse(CallResponse&& resp) { SetFailed(status); return; } + // End the OpenTelemetry span with success status + if (otel_span_.IsActive()) { + otel_span_.SetStatus(true, "OK"); + otel_span_.End(); + } if (SetState(RpcCallState::FINISHED_SUCCESS)) { InvokeCallback(now); } @@ -598,6 +618,13 @@ void OutboundCall::SetFinished() { outbound_call_metrics_->time_to_response->Increment( MicrosecondsSinceStart(CoarseMonoClock::Now())); } + + // End the OpenTelemetry span with success status + if (otel_span_.IsActive()) { + otel_span_.SetStatus(true, "OK"); + otel_span_.End(); + } + if (SetState(RpcCallState::FINISHED_SUCCESS)) { InvokeCallback(); } @@ -605,6 +632,14 @@ void OutboundCall::SetFinished() { void OutboundCall::SetFailed(const Status &status, std::unique_ptr err_pb) { TRACE_TO(trace_, "Call Failed."); + + // End the OpenTelemetry span with error status + if (otel_span_.IsActive()) { + otel_span_.SetStatus(false, status.ToString()); + otel_span_.SetAttribute("rpc.error", status.CodeAsString()); + otel_span_.End(); + } + bool invoke_callback; { std::lock_guard l(mtx_); @@ -636,6 +671,14 @@ void OutboundCall::SetFailed(const Status &status, std::unique_ptr connection_weak_; + // OpenTelemetry span for distributed tracing. Created when the call starts, ended when complete. + OtelSpanHandle otel_span_; + // InvokeCallbackTask should be able to call InvokeCallbackSync and we don't want other that // method to be public. friend class InvokeCallbackTask; diff --git a/src/yb/rpc/rpc_controller.h b/src/yb/rpc/rpc_controller.h index 155dc418ce61..f392906af734 100644 --- a/src/yb/rpc/rpc_controller.h +++ b/src/yb/rpc/rpc_controller.h @@ -32,6 +32,7 @@ #pragma once #include +#include #include "yb/util/logging.h" @@ -139,6 +140,11 @@ class RpcController { InvokeCallbackMode invoke_callback_mode() { return invoke_callback_mode_; } + // Set the traceparent for distributed tracing. If set, the RPC will create + // a child span under this traceparent. If not set, no tracing span is created. + void set_traceparent(const std::string& traceparent) { traceparent_ = traceparent; } + const std::string& traceparent() const { return traceparent_; } + // Return the configured timeout. MonoDelta timeout() const; @@ -183,6 +189,7 @@ class RpcController { std::unique_ptr outbound_sidecars_; bool TEST_disable_outbound_call_response_processing = false; + std::string traceparent_; DISALLOW_COPY_AND_ASSIGN(RpcController); }; diff --git a/src/yb/tserver/tablet_server_main_impl.cc b/src/yb/tserver/tablet_server_main_impl.cc index e1082f53f645..8b5cda28ff90 100644 --- a/src/yb/tserver/tablet_server_main_impl.cc +++ b/src/yb/tserver/tablet_server_main_impl.cc @@ -66,6 +66,7 @@ #include "yb/util/logging.h" #include "yb/util/main_util.h" #include "yb/util/mem_tracker.h" +#include "yb/util/otel_tracing.h" #include "yb/util/port_picker.h" #include "yb/util/result.h" #include "yb/util/size_literals.h" @@ -248,6 +249,9 @@ int TabletServerMain(int argc, char** argv) { LOG_AND_RETURN_FROM_MAIN_NOT_OK(MasterTServerParseFlagsAndInit( TabletServerOptions::kServerType, /*is_master=*/false, &argc, &argv)); + // Initialize OpenTelemetry tracing (if enabled via environment variables) + LOG_AND_RETURN_FROM_MAIN_NOT_OK(OtelTracing::InitFromEnv("yb-tserver")); + auto termination_monitor = TerminationMonitor::Create(); SetProxyAddresses(); @@ -437,6 +441,9 @@ int TabletServerMain(int argc, char** argv) { LOG(WARNING) << "Stopping Tablet server"; server->Shutdown(); + // Shutdown OpenTelemetry tracing + OtelTracing::Shutdown(); + return EXIT_SUCCESS; } diff --git a/src/yb/util/CMakeLists.txt b/src/yb/util/CMakeLists.txt index de3201999187..c73dc21891c0 100644 --- a/src/yb/util/CMakeLists.txt +++ b/src/yb/util/CMakeLists.txt @@ -183,6 +183,8 @@ set(UTIL_SRCS oid_generator.cc once.cc operation_counter.cc + otel_tracing.cc + otel_http_exporter.cc os-util.cc path_util.cc pb_util-internal.cc @@ -265,6 +267,65 @@ set(UTIL_LIBS ${OPENSSL_CRYPTO_LIBRARY} ${OPENSSL_SSL_LIBRARY}) +# Add OpenTelemetry static libraries +# OpenTelemetry must be built from source with static libraries. +# See build-support/build_opentelemetry.sh for the build script. +# Look for OpenTelemetry in standard locations +# First check thirdparty/installed (default for build_opentelemetry.sh) +set(OTEL_SEARCH_PATHS + "${YB_SRC_ROOT}/thirdparty/installed/opentelemetry" + "/usr/local/opentelemetry" + "/opt/opentelemetry" + "$ENV{HOME}/.local/opentelemetry") + +foreach(SEARCH_PATH ${OTEL_SEARCH_PATHS}) + if(EXISTS "${SEARCH_PATH}/include/opentelemetry/version.h") + set(OTEL_ROOT "${SEARCH_PATH}") + break() + endif() +endforeach() + +if(OTEL_ROOT) + message(STATUS "Found OpenTelemetry at ${OTEL_ROOT}") + include_directories(SYSTEM "${OTEL_ROOT}/include") + + # Find required static libraries (using ostream exporter for simplicity) + find_library(OTEL_TRACE_LIB + NAMES libopentelemetry_trace.a + PATHS "${OTEL_ROOT}/lib" + NO_DEFAULT_PATH) + find_library(OTEL_COMMON_LIB + NAMES libopentelemetry_common.a + PATHS "${OTEL_ROOT}/lib" + NO_DEFAULT_PATH) + find_library(OTEL_RESOURCES_LIB + NAMES libopentelemetry_resources.a + PATHS "${OTEL_ROOT}/lib" + NO_DEFAULT_PATH) + find_library(OTEL_EXPORTER_OSTREAM_LIB + NAMES libopentelemetry_exporter_ostream_span.a + PATHS "${OTEL_ROOT}/lib" + NO_DEFAULT_PATH) + + if(OTEL_TRACE_LIB AND OTEL_COMMON_LIB AND OTEL_EXPORTER_OSTREAM_LIB) + message(STATUS "OpenTelemetry static libraries found - OTEL support enabled") + + # Use ostream exporter (logs spans to stderr) to avoid complex dependencies + # This is a simple, reliable exporter with no external dependencies + list(APPEND UTIL_LIBS + ${OTEL_EXPORTER_OSTREAM_LIB} + ${OTEL_TRACE_LIB} + ${OTEL_RESOURCES_LIB} + ${OTEL_COMMON_LIB}) + else() + message(STATUS "OpenTelemetry static libraries not found - OTEL support disabled") + message(STATUS " To enable OTEL: run build-support/build_opentelemetry.sh") + endif() +else() + message(STATUS "OpenTelemetry not found - OTEL support disabled") + message(STATUS " To enable OTEL: run build-support/build_opentelemetry.sh") +endif() + if(NOT APPLE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DROCKSDB_FALLOCATE_PRESENT") set(UTIL_LIBS diff --git a/src/yb/util/otel_http_exporter.cc b/src/yb/util/otel_http_exporter.cc new file mode 100644 index 000000000000..04aff9117144 --- /dev/null +++ b/src/yb/util/otel_http_exporter.cc @@ -0,0 +1,210 @@ +// Copyright (c) YugabyteDB, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. +// + +#include "yb/util/otel_http_exporter.h" + +#include +#include +#include + +#include "yb/util/curl_util.h" +#include "yb/util/faststring.h" +#include "yb/util/logging.h" +#include "yb/util/status.h" + +namespace yb { + +namespace { + +// Global singleton instance +SimpleOtlpHttpSender g_otlp_sender; + +} // namespace + +SimpleOtlpHttpSender::SimpleOtlpHttpSender() + : enabled_(false) { + const char* service_name_env = std::getenv("OTEL_SERVICE_NAME"); + service_name_ = (service_name_env && service_name_env[0] != '\0') ? service_name_env : "yugabyte"; +} + +SimpleOtlpHttpSender::~SimpleOtlpHttpSender() = default; + +void SimpleOtlpHttpSender::SetEndpoint(const std::string& endpoint) { + endpoint_ = endpoint; + enabled_.store(!endpoint_.empty(), std::memory_order_release); +} + +void SimpleOtlpHttpSender::SetServiceName(const std::string& service_name) { + service_name_ = service_name; +} + +bool SimpleOtlpHttpSender::IsEnabled() const { + return enabled_.load(std::memory_order_acquire); +} + +bool SimpleOtlpHttpSender::SendSpan(const SimpleSpanData& span) { + std::vector spans; + spans.push_back(span); + return SendSpans(spans); +} + +bool SimpleOtlpHttpSender::SendSpans(const std::vector& spans) { + if (!IsEnabled() || spans.empty()) { + return true; + } + + std::string json_payload = SpansToJson(spans); + + LOG(INFO) << "[OTEL] Sending " << spans.size() << " span(s) to " << endpoint_; + VLOG(3) << "[OTEL] Payload: " << json_payload; + + try { + EasyCurl curl; + faststring response; + + auto status = curl.PostToURL( + endpoint_, + json_payload, + "application/json", + &response, + 5 // 5 second timeout + ); + + if (!status.ok()) { + LOG(WARNING) << "[OTEL] HTTP POST to " << endpoint_ << " failed: " << status.ToString(); + return false; + } + + LOG(INFO) << "[OTEL] Successfully sent " << spans.size() << " span(s), response: " + << response.size() << " bytes"; + return true; + + } catch (const std::exception& e) { + LOG(ERROR) << "[OTEL] Exception sending to collector: " << e.what(); + return false; + } +} + +std::string SimpleOtlpHttpSender::JsonEscape(const std::string& str) { + std::ostringstream oss; + for (size_t i = 0; i < str.size(); ++i) { + unsigned char c = static_cast(str[i]); + switch (c) { + case '"': oss << "\\\""; break; + case '\\': oss << "\\\\"; break; + case '\b': oss << "\\b"; break; + case '\f': oss << "\\f"; break; + case '\n': oss << "\\n"; break; + case '\r': oss << "\\r"; break; + case '\t': oss << "\\t"; break; + default: + if (c < 0x20) { + oss << "\\u" << std::hex << std::setw(4) << std::setfill('0') << static_cast(c); + } else { + oss << c; + } + break; + } + } + return oss.str(); +} + +std::string SimpleOtlpHttpSender::SpansToJson(const std::vector& spans) const { + std::ostringstream json; + + json << R"({"resourceSpans":[{"resource":{"attributes":[)"; + json << R"({"key":"service.name","value":{"stringValue":")" << JsonEscape(service_name_) << R"("}},)"; + json << R"({"key":"telemetry.sdk.language","value":{"stringValue":"cpp"}},)"; + json << R"({"key":"telemetry.sdk.name","value":{"stringValue":"yugabyte-simple"}})"; + json << R"(]},"scopeSpans":[{"scope":{"name":"yugabyte","version":"1.0.0"},"spans":[)"; + + bool first_span = true; + for (size_t i = 0; i < spans.size(); ++i) { + const SimpleSpanData& span = spans[i]; + + if (!first_span) { + json << ","; + } + first_span = false; + + json << "{"; + json << R"("traceId":")" << span.trace_id << R"(",)"; + json << R"("spanId":")" << span.span_id << R"(",)"; + + if (!span.parent_span_id.empty()) { + json << R"("parentSpanId":")" << span.parent_span_id << R"(",)"; + } + + json << R"("name":")" << JsonEscape(span.name) << R"(",)"; + json << R"("kind":)" << span.kind << ","; + json << R"("startTimeUnixNano":")" << span.start_time_ns << R"(",)"; + json << R"("endTimeUnixNano":")" << span.end_time_ns << R"(",)"; + + json << R"("attributes":[)"; + bool first_attr = true; + for (size_t j = 0; j < span.string_attributes.size(); ++j) { + if (!first_attr) json << ","; + first_attr = false; + json << R"({"key":")" << JsonEscape(span.string_attributes[j].first) + << R"(","value":{"stringValue":")" << JsonEscape(span.string_attributes[j].second) << R"("}})"; + } + for (size_t j = 0; j < span.int_attributes.size(); ++j) { + if (!first_attr) json << ","; + first_attr = false; + json << R"({"key":")" << JsonEscape(span.int_attributes[j].first) + << R"(","value":{"intValue":")" << span.int_attributes[j].second << R"("}})"; + } + json << "],"; + + json << R"("status":{"code":)" << span.status_code; + if (!span.status_message.empty()) { + json << R"(,"message":")" << JsonEscape(span.status_message) << R"(")"; + } + json << "}}"; + } + + json << "]}]}]}"; + return json.str(); +} + +SimpleOtlpHttpSender& GetGlobalOtlpSender() { + return g_otlp_sender; +} + +void InitGlobalOtlpSenderFromEnv(const std::string& service_name) { + std::string endpoint; + const char* endpoint_env = std::getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); + + if (endpoint_env && endpoint_env[0] != '\0') { + endpoint = endpoint_env; + if (endpoint.find("/v1/traces") == std::string::npos) { + if (!endpoint.empty() && endpoint.back() == '/') { + endpoint.pop_back(); + } + endpoint += "/v1/traces"; + } + } else { + endpoint = "http://localhost:4318/v1/traces"; + } + + g_otlp_sender.SetEndpoint(endpoint); + LOG(INFO) << "[OTEL] HTTP exporter endpoint: " << endpoint; + + const char* service_name_env = std::getenv("OTEL_SERVICE_NAME"); + std::string resolved_service_name = (service_name_env && service_name_env[0] != '\0') + ? service_name_env : "yugabyte"; + g_otlp_sender.SetServiceName(resolved_service_name); + LOG(INFO) << "[OTEL] HTTP exporter service name: " << resolved_service_name; +} + +} // namespace yb diff --git a/src/yb/util/otel_http_exporter.h b/src/yb/util/otel_http_exporter.h new file mode 100644 index 000000000000..8c61da481dcb --- /dev/null +++ b/src/yb/util/otel_http_exporter.h @@ -0,0 +1,95 @@ +// Copyright (c) YugabyteDB, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. +// + +#pragma once + +#include +#include +#include +#include +#include + +namespace yb { + +// Simple span data structure - no OTEL SDK dependencies. +// This allows us to capture span data without pulling in any OTEL headers. +struct SimpleSpanData { + std::string trace_id; // 32-char hex string + std::string span_id; // 16-char hex string + std::string parent_span_id; // 16-char hex string or empty + std::string name; + int64_t start_time_ns; // Nanoseconds since Unix epoch + int64_t end_time_ns; // Nanoseconds since Unix epoch + int kind; // 1=internal, 2=server, 3=client, 4=producer, 5=consumer + int status_code; // 0=unset, 1=ok, 2=error + std::string status_message; + std::vector > string_attributes; + std::vector > int_attributes; + + SimpleSpanData() + : start_time_ns(0), end_time_ns(0), kind(1), status_code(0) {} +}; + +// Minimal OTLP/HTTP JSON sender for OpenTelemetry traces. +// This class has ZERO OTEL SDK dependencies - it works entirely with SimpleSpanData +// and standard C++ types, avoiding any potential ABI/template issues. +// +// Usage: +// SimpleOtlpHttpSender sender; +// sender.SetServiceName("my-service"); +// sender.SetEndpoint("http://localhost:4318/v1/traces"); +// +// SimpleSpanData span; +// span.trace_id = "..."; +// // ... fill in other fields +// sender.SendSpan(span); +// +class SimpleOtlpHttpSender { + public: + SimpleOtlpHttpSender(); + ~SimpleOtlpHttpSender(); + + // Configuration + void SetEndpoint(const std::string& endpoint); + void SetServiceName(const std::string& service_name); + + // Send a single span to the collector. Returns true on success. + bool SendSpan(const SimpleSpanData& span); + + // Send multiple spans in a single request. Returns true on success. + bool SendSpans(const std::vector& spans); + + // Check if the sender is enabled (endpoint is set) + bool IsEnabled() const; + + private: + // Convert spans to OTLP/HTTP JSON format + std::string SpansToJson(const std::vector& spans) const; + + // Escape a string for JSON + static std::string JsonEscape(const std::string& str); + + std::string endpoint_; + std::string service_name_; + std::atomic enabled_; +}; + +// Global singleton accessor for convenience +SimpleOtlpHttpSender& GetGlobalOtlpSender(); + +// Initialize the global sender from environment variables: +// - OTEL_EXPORTER_OTLP_ENDPOINT: Base endpoint (default: http://localhost:4318) +// The service_name parameter should be the already-resolved service name. +void InitGlobalOtlpSenderFromEnv(const std::string& service_name); + +} // namespace yb diff --git a/src/yb/util/otel_tracing.cc b/src/yb/util/otel_tracing.cc new file mode 100644 index 000000000000..5899063ac188 --- /dev/null +++ b/src/yb/util/otel_tracing.cc @@ -0,0 +1,449 @@ +// Copyright (c) YugabyteDB, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. +// + +#include "yb/util/otel_tracing.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "yb/util/logging.h" +#include "yb/util/otel_http_exporter.h" +#include "yb/util/status.h" + +// ------------------------------------------------------------------------------------------------- +// Bridging stubs for PostgreSQL traceparent helpers +// +// The PostgreSQL backend defines real C functions with these names in +// src/postgres/src/backend/utils/misc/pg_yb_utils.c. We also link a number of +// non-Postgres tools (e.g. log-dump, protoc-gen-yrpc) against libyb_pggate.so, +// which declares these symbols but does not link against the Postgres backend. +// +// Because YugabyteDB executables are linked with -Wl,--no-allow-shlib-undefined, +// any shared library on the link line must have all of its undefined references +// resolved. To avoid forcing every non-Postgres tool to link against the +// Postgres backend library, we provide weak, no-op fallbacks here in libyb_util. +// +// When the Postgres backend is linked into a process, its strong definitions of +// these functions override these weak stubs, so query-level traceparent +// propagation continues to work as intended. +// ------------------------------------------------------------------------------------------------- +extern "C" { + +__attribute__((weak)) const char* YbGetCurrentTraceparent(void) { + // No active traceparent in non-Postgres contexts. + return ""; +} + +__attribute__((weak)) void YbClearTraceparent(void) { + // No-op when running outside the Postgres backend. +} + +} // extern "C" + +namespace nostd = opentelemetry::nostd; +namespace trace_api = opentelemetry::trace; +namespace trace_sdk = opentelemetry::sdk::trace; +namespace ostream_exporter = opentelemetry::exporter::trace; + +namespace yb { + +namespace otel_internal { + +struct SpanContext { + bool active = false; + nostd::shared_ptr span; + nostd::shared_ptr scope; + + std::string name; + std::string parent_span_id; + int64_t start_time_ns = 0; + int span_kind = 1; + int status_code = 0; + std::string status_message; + std::vector > string_attributes; + std::vector > int_attributes; +}; + +} // namespace otel_internal + +namespace { + +// Global state +std::atomic g_otel_enabled{false}; +std::string g_service_name; +std::mutex g_init_mutex; +nostd::shared_ptr g_tracer_provider; +nostd::shared_ptr g_tracer; + +// Helper to convert trace ID to 32-char hex string +std::string TraceIdToHex(const trace_api::TraceId& trace_id) { + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + for (size_t i = 0; i < trace_api::TraceId::kSize; ++i) { + oss << std::setw(2) << static_cast(trace_id.Id()[i]); + } + return oss.str(); +} + +// Helper to convert span ID to 16-char hex string +std::string SpanIdToHex(const trace_api::SpanId& span_id) { + std::ostringstream oss; + oss << std::hex << std::setfill('0'); + for (size_t i = 0; i < trace_api::SpanId::kSize; ++i) { + oss << std::setw(2) << static_cast(span_id.Id()[i]); + } + return oss.str(); +} + +// Get current time in nanoseconds since Unix epoch +int64_t GetCurrentTimeNanos() { + auto now = std::chrono::system_clock::now(); + auto duration = now.time_since_epoch(); + return std::chrono::duration_cast(duration).count(); +} + +} // anonymous namespace + +OtelSpanHandle::OtelSpanHandle() : context_(nullptr) {} + +OtelSpanHandle::OtelSpanHandle(std::unique_ptr context) + : context_(std::move(context)) {} + +OtelSpanHandle::~OtelSpanHandle() { + End(); // Safety net - end span if not already ended +} + +OtelSpanHandle::OtelSpanHandle(OtelSpanHandle&& other) noexcept + : context_(std::move(other.context_)) {} + +OtelSpanHandle& OtelSpanHandle::operator=(OtelSpanHandle&& other) noexcept { + if (this != &other) { + End(); // End current span before replacing + context_ = std::move(other.context_); + } + return *this; +} + +void OtelSpanHandle::End() { + if (!context_ || !context_->active || !context_->span) { + return; + } + + // Capture end time and send span data + if (GetGlobalOtlpSender().IsEnabled()) { + SimpleSpanData span_data; + auto span_context = context_->span->GetContext(); + span_data.trace_id = TraceIdToHex(span_context.trace_id()); + span_data.span_id = SpanIdToHex(span_context.span_id()); + span_data.parent_span_id = context_->parent_span_id; + span_data.name = context_->name; + span_data.start_time_ns = context_->start_time_ns; + span_data.end_time_ns = GetCurrentTimeNanos(); + span_data.kind = context_->span_kind; + span_data.status_code = context_->status_code; + span_data.status_message = context_->status_message; + span_data.string_attributes = context_->string_attributes; + span_data.int_attributes = context_->int_attributes; + + if (!GetGlobalOtlpSender().SendSpan(span_data)) { + LOG(WARNING) << "[OTEL] Failed to send span via HTTP: " << span_data.name; + } + } + + context_->span->End(); + context_->active = false; // Mark as ended so End() is idempotent +} + +void OtelSpanHandle::SetStatus(bool ok, const std::string& description) { + if (!context_ || !context_->active) return; + + context_->status_code = ok ? 1 : 2; + context_->status_message = description; + + if (context_->span) { + if (ok) { + context_->span->SetStatus(trace_api::StatusCode::kOk, description); + } else { + context_->span->SetStatus(trace_api::StatusCode::kError, description); + } + } +} + +void OtelSpanHandle::SetAttribute(const std::string& key, const std::string& value) { + if (!context_ || !context_->active) return; + + context_->string_attributes.push_back(std::make_pair(key, value)); + + if (context_->span) { + context_->span->SetAttribute(key, value); + } +} + +void OtelSpanHandle::SetAttribute(const std::string& key, int64_t value) { + if (!context_ || !context_->active) return; + + context_->int_attributes.push_back(std::make_pair(key, value)); + + if (context_->span) { + context_->span->SetAttribute(key, value); + } +} + +void OtelSpanHandle::SetAttribute(const std::string& key, double value) { + if (!context_ || !context_->active) return; + if (context_->span) { + context_->span->SetAttribute(key, value); + } +} + +void OtelSpanHandle::SetAttribute(const std::string& key, bool value) { + if (!context_ || !context_->active) return; + if (context_->span) { + context_->span->SetAttribute(key, value); + } +} + +bool OtelSpanHandle::IsActive() const { + return context_ && context_->active; +} + +Status OtelTracing::InitFromEnv(const std::string& default_service_name) { + std::lock_guard lock(g_init_mutex); + + g_service_name = default_service_name; + + const char* debug_env = std::getenv("OTEL_EXPORTER_DEBUG"); + bool use_debug_exporter = debug_env && (std::string(debug_env) == "true" || + std::string(debug_env) == "1" || + std::string(debug_env) == "TRUE"); + + if (use_debug_exporter) { + LOG(INFO) << "[OTEL] Debug mode enabled (OTEL_EXPORTER_DEBUG=true), using OStream exporter"; + } else { + InitGlobalOtlpSenderFromEnv(g_service_name); + } + + try { + auto exporter = ostream_exporter::OStreamSpanExporterFactory::Create(); + if (!exporter) { + return STATUS(RuntimeError, "Failed to create ostream exporter"); + } + + auto processor = trace_sdk::SimpleSpanProcessorFactory::Create(std::move(exporter)); + if (!processor) { + return STATUS(RuntimeError, "Failed to create span processor"); + } + + auto provider = trace_sdk::TracerProviderFactory::Create(std::move(processor)); + if (!provider) { + return STATUS(RuntimeError, "Failed to create tracer provider"); + } + + g_tracer_provider = nostd::shared_ptr(provider.release()); + trace_api::Provider::SetTracerProvider(g_tracer_provider); + + g_tracer = g_tracer_provider->GetTracer(g_service_name, "1.0.0"); + if (!g_tracer) { + return STATUS(RuntimeError, "Failed to get tracer"); + } + + auto propagator = nostd::shared_ptr( + new opentelemetry::trace::propagation::HttpTraceContext()); + opentelemetry::context::propagation::GlobalTextMapPropagator::SetGlobalPropagator(propagator); + + g_otel_enabled.store(true, std::memory_order_release); + + if (use_debug_exporter) { + LOG(INFO) << "[OTEL] Using OStream exporter (debug mode) - spans will be written to stdout"; + } else if (GetGlobalOtlpSender().IsEnabled()) { + LOG(INFO) << "[OTEL] HTTP exporter is enabled, spans will be sent via HTTP"; + } else { + LOG(INFO) << "[OTEL] HTTP exporter not configured (set OTEL_EXPORTER_OTLP_ENDPOINT)"; + } + + LOG(INFO) << "[OTEL] OpenTelemetry tracing initialized successfully"; + + } catch (const std::exception& e) { + return STATUS_FORMAT(RuntimeError, "Failed to initialize OpenTelemetry: $0", e.what()); + } + + return Status::OK(); +} + +void OtelTracing::Shutdown() { + std::lock_guard lock(g_init_mutex); + + if (g_otel_enabled.load(std::memory_order_acquire)) { + LOG(INFO) << "Shutting down OpenTelemetry tracing"; + if (g_tracer_provider) { + auto* raw_provider = g_tracer_provider.get(); + if (auto* sdk_provider = dynamic_cast(raw_provider)) { + sdk_provider->ForceFlush(); + } + } + g_tracer = nullptr; + g_tracer_provider = nullptr; + g_otel_enabled.store(false, std::memory_order_release); + } +} + +bool OtelTracing::IsEnabled() { + return g_otel_enabled.load(std::memory_order_acquire); +} + +OtelSpanHandle OtelTracing::StartSpan(const std::string& name) { + if (!IsEnabled()) { + return OtelSpanHandle(); + } + + auto context = std::make_unique(); + context->active = true; + context->name = name; + context->start_time_ns = GetCurrentTimeNanos(); + context->span_kind = 1; // internal + + if (g_tracer) { + auto current_span = trace_api::Tracer::GetCurrentSpan(); + if (current_span && current_span->GetContext().IsValid()) { + context->parent_span_id = SpanIdToHex(current_span->GetContext().span_id()); + trace_api::StartSpanOptions options; + options.parent = current_span->GetContext(); + context->span = g_tracer->StartSpan(name, options); + } else { + context->span = g_tracer->StartSpan(name); + } + } + + return OtelSpanHandle(std::move(context)); +} + +OtelSpanHandle OtelTracing::StartSpanFromTraceparent( + const std::string& name, + const std::string& traceparent) { + if (!IsEnabled() || traceparent.empty()) { + return OtelSpanHandle(); + } + + if (traceparent.length() < 55 || traceparent[2] != '-' || traceparent[35] != '-' || + traceparent[52] != '-') { + LOG(WARNING) << "[OTEL] Invalid traceparent format: " << traceparent; + return OtelSpanHandle(); + } + + std::string parent_span_id = traceparent.substr(36, 16); + + auto context = std::make_unique(); + context->active = true; + context->name = name; + context->parent_span_id = parent_span_id; + context->start_time_ns = GetCurrentTimeNanos(); + context->span_kind = 2; + + if (g_tracer) { + class TraceparentCarrier : public opentelemetry::context::propagation::TextMapCarrier { + public: + explicit TraceparentCarrier(const std::string& traceparent_value) + : traceparent_(traceparent_value) {} + + nostd::string_view Get(nostd::string_view key) const noexcept override { + if (key == "traceparent") { + return nostd::string_view(traceparent_); + } + return ""; + } + + void Set(nostd::string_view key, nostd::string_view value) noexcept override { + if (key == "traceparent") { + traceparent_ = std::string(value); + } + } + + private: + std::string traceparent_; + }; + + TraceparentCarrier carrier(traceparent); + auto propagator = opentelemetry::context::propagation::GlobalTextMapPropagator::GetGlobalPropagator(); + auto current_ctx = opentelemetry::context::RuntimeContext::GetCurrent(); + auto new_ctx = propagator->Extract(carrier, current_ctx); + + auto remote_span = opentelemetry::trace::GetSpan(new_ctx); + auto remote_context = remote_span->GetContext(); + + if (remote_context.IsValid()) { + trace_api::StartSpanOptions options; + options.parent = remote_context; + options.kind = trace_api::SpanKind::kServer; + context->span = g_tracer->StartSpan(name, options); + } else { + context->span = g_tracer->StartSpan(name); + } + } + + return OtelSpanHandle(std::move(context)); +} + +void OtelTracing::AdoptSpan(const OtelSpanHandle& span) { + if (!span.IsActive() || !span.context_ || !span.context_->span) { + return; + } + span.context_->scope = nostd::shared_ptr( + new trace_api::Scope(span.context_->span)); +} + +void OtelTracing::EndCurrentSpan() { +} + +bool OtelTracing::HasActiveContext() { + if (!IsEnabled()) { + return false; + } + auto current_span = trace_api::Tracer::GetCurrentSpan(); + return current_span && current_span->GetContext().IsValid(); +} + +std::string OtelTracing::GetCurrentTraceparent() { + if (!IsEnabled()) { + return ""; + } + auto current_span = trace_api::Tracer::GetCurrentSpan(); + if (!current_span || !current_span->GetContext().IsValid()) { + return ""; + } + + auto span_context = current_span->GetContext(); + std::string trace_id = TraceIdToHex(span_context.trace_id()); + std::string span_id = SpanIdToHex(span_context.span_id()); + std::string flags = span_context.IsSampled() ? "01" : "00"; + + return "00-" + trace_id + "-" + span_id + "-" + flags; +} + +} // namespace yb diff --git a/src/yb/util/otel_tracing.h b/src/yb/util/otel_tracing.h new file mode 100644 index 000000000000..1691456104c3 --- /dev/null +++ b/src/yb/util/otel_tracing.h @@ -0,0 +1,103 @@ +// Copyright (c) YugabyteDB, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations +// under the License. +// + +#pragma once + +#include +#include +#include + +#include "yb/util/status.h" + +namespace yb { + +namespace otel_internal { +struct SpanContext; +} // namespace otel_internal + +class OtelSpanHandle { + public: + OtelSpanHandle(); + ~OtelSpanHandle(); + + // Move-only type + OtelSpanHandle(OtelSpanHandle&& other) noexcept; + OtelSpanHandle& operator=(OtelSpanHandle&& other) noexcept; + + OtelSpanHandle(const OtelSpanHandle&) = delete; + OtelSpanHandle& operator=(const OtelSpanHandle&) = delete; + + void SetStatus(bool ok, const std::string& description = ""); + + void SetAttribute(const std::string& key, const std::string& value); + void SetAttribute(const std::string& key, int64_t value); + void SetAttribute(const std::string& key, double value); + void SetAttribute(const std::string& key, bool value); + + // Explicitly end the span. Safe to call multiple times. + void End(); + + bool IsActive() const; + + private: + friend class OtelTracing; + explicit OtelSpanHandle(std::unique_ptr context); + + std::unique_ptr context_; +}; + +class OtelTracing { + public: + static Status InitFromEnv(const std::string& default_service_name); + static void Shutdown(); + static bool IsEnabled(); + static OtelSpanHandle StartSpan(const std::string& name); + static OtelSpanHandle StartSpanFromTraceparent( + const std::string& name, + const std::string& traceparent); + static void AdoptSpan(const OtelSpanHandle& span); + static void EndCurrentSpan(); + static bool HasActiveContext(); + static std::string GetCurrentTraceparent(); + + private: + OtelTracing() = delete; +}; + +class ScopedOtelSpan { + public: + explicit ScopedOtelSpan(OtelSpanHandle&& span) + : span_(std::move(span)) { + if (span_.IsActive()) { + OtelTracing::AdoptSpan(span_); + } + } + + ~ScopedOtelSpan() { + if (span_.IsActive()) { + OtelTracing::EndCurrentSpan(); + } + } + + ScopedOtelSpan(const ScopedOtelSpan&) = delete; + ScopedOtelSpan& operator=(const ScopedOtelSpan&) = delete; + + OtelSpanHandle& span() { return span_; } + const OtelSpanHandle& span() const { return span_; } + + private: + OtelSpanHandle span_; +}; + +} // namespace yb + diff --git a/src/yb/yql/pggate/pg_client.cc b/src/yb/yql/pggate/pg_client.cc index 65370993d549..b89805cc331d 100644 --- a/src/yb/yql/pggate/pg_client.cc +++ b/src/yb/yql/pggate/pg_client.cc @@ -50,6 +50,13 @@ #include "yb/yql/pggate/pggate_flags.h" #include "yb/yql/pggate/util/ybc_guc.h" +#include "yb/util/otel_tracing.h" + +// Forward declarations for Postgres C functions to access traceparent +extern "C" { + const char* YbGetCurrentTraceparent(void); +} + DECLARE_bool(enable_object_lock_fastpath); DECLARE_bool(use_node_hostname_for_local_tserver); DECLARE_int32(backfill_index_client_rpc_timeout_ms); @@ -606,6 +613,17 @@ class PgClient::Impl : public BigDataFetcher { Result OpenTable( const PgObjectId& table_id, bool reopen, uint64_t min_ysql_catalog_version, master::IncludeHidden include_hidden) { + OtelSpanHandle open_table_span; + std::unique_ptr scoped_span; + if (OtelTracing::HasActiveContext()) { + open_table_span = OtelTracing::StartSpan("pggate.open_table"); + if (open_table_span.IsActive()) { + open_table_span.SetAttribute("db.table_oid", static_cast(table_id.object_oid)); + open_table_span.SetAttribute("db.database_oid", static_cast(table_id.database_oid)); + scoped_span = std::make_unique(std::move(open_table_span)); + } + } + tserver::PgOpenTableRequestPB req; req.set_table_id(table_id.GetYbTableId()); req.set_reopen(reopen); @@ -622,6 +640,11 @@ class PgClient::Impl : public BigDataFetcher { auto result = make_scoped_refptr( table_id, resp.info(), BuildTablePartitionList(resp.partitions(), table_id)); RETURN_NOT_OK(result->Init()); + + if (scoped_span && scoped_span->span().IsActive()) { + scoped_span->span().SetAttribute("db.table", result->table_name().table_name()); + } + return result; } @@ -1638,6 +1661,11 @@ class PgClient::Impl : public BigDataFetcher { } else { controller->set_timeout(timeout_); } + const char* c_traceparent = YbGetCurrentTraceparent(); + if (c_traceparent && c_traceparent[0] != '\0') { + controller->set_traceparent(c_traceparent); + } + return controller; } diff --git a/src/yb/yql/pggate/pg_session.cc b/src/yb/yql/pggate/pg_session.cc index 4bd31356b930..cf2dcd1e96e8 100644 --- a/src/yb/yql/pggate/pg_session.cc +++ b/src/yb/yql/pggate/pg_session.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include "yb/client/table_info.h" @@ -33,6 +34,7 @@ #include "yb/util/format.h" #include "yb/util/logging.h" #include "yb/util/oid_generator.h" +#include "yb/util/otel_tracing.h" #include "yb/util/result.h" #include "yb/util/status_format.h" @@ -420,6 +422,11 @@ class PgSession::RunHelper { }); } + // Check if there are operations that will be flushed (not just buffered) + bool HasOpsToFlush() const { + return !ops_info_.ops.Empty(); + } + private: inline static bool IsTransactional(SessionType type) { return type == SessionType::kTransactional; @@ -768,6 +775,18 @@ Result PgSession::FlushOperations(BufferableOperations&& ops, bool << " session (num ops: " << ops.Size() << ")"; } + // Create a batch span for buffered operations being flushed + std::unique_ptr scoped_batch_span; + if (OtelTracing::HasActiveContext()) { + auto batch_span = OtelTracing::StartSpan("pggate.batch"); + if (batch_span.IsActive()) { + batch_span.SetAttribute("yb.session_type", transactional ? "transactional" : "regular"); + batch_span.SetAttribute("yb.buffered", true); + batch_span.SetAttribute("yb.op_count", static_cast(ops.Size())); + scoped_batch_span = std::make_unique(std::move(batch_span)); + } + } + if (transactional) { RETURN_NOT_OK(pg_txn_manager_->CalculateIsolation( false /* read_only */, @@ -795,6 +814,7 @@ Result PgSession::FlushOperations(BufferableOperations&& ops, bool Result PgSession::Perform(BufferableOperations&& ops, PerformOptions&& ops_options) { DCHECK(!ops.Empty()); + tserver::PgPerformOptionsPB options; if (ops_options.use_catalog_session) { if (catalog_read_time_) { @@ -1110,13 +1130,23 @@ Result PgSession::DoRunAsync( RunHelper runner( this, group_session_type, in_txn_limit, force_non_bufferable); const auto ddl_force_catalog_mod_opt = pg_txn_manager_->GetDdlForceCatalogModification(); + + int64_t catalog_read_count = 0; + int64_t catalog_write_count = 0; + int64_t user_read_count = 0; + int64_t user_write_count = 0; + std::string table_names; + std::set seen_tables; + auto processor = [this, is_ddl = ddl_force_catalog_mod_opt.has_value(), force_catalog_modification = is_major_pg_version_upgrade_ || ddl_force_catalog_mod_opt.value_or(false) || YBIsMajorUpgradeInitDb(), - group_session_type, &runner, non_ddl_txn_for_sys_tables_allowed](const auto& table_op) { + group_session_type, &runner, non_ddl_txn_for_sys_tables_allowed, + &catalog_read_count, &catalog_write_count, &user_read_count, &user_write_count, + &table_names, &seen_tables](const auto& table_op) { DCHECK(!table_op.IsEmpty()); const auto& table = *table_op.table; const auto& op = *table_op.operation; @@ -1137,12 +1167,59 @@ Result PgSession::DoRunAsync( has_catalog_write_ops_in_ddl_mode_ = has_catalog_write_ops_in_ddl_mode_ || (is_ddl && op->is_write() && is_ysql_catalog_table); + + if (is_ysql_catalog_table) { + if (op->is_read()) { + catalog_read_count++; + } else { + catalog_write_count++; + } + } else { + if (op->is_read()) { + user_read_count++; + } else { + user_write_count++; + } + } + const auto& tbl_name = table.table_name().table_name(); + if (seen_tables.find(tbl_name) == seen_tables.end()) { + seen_tables.insert(tbl_name); + if (!table_names.empty()) { + table_names += ","; + } + table_names += tbl_name; + } + return runner.Apply(table, op); }; RETURN_NOT_OK(processor(first_table_op)); for (; !table_op.IsEmpty(); table_op = generator()) { RETURN_NOT_OK(processor(table_op)); } + + // Only create batch span if we're actually going to flush (not just buffer) + // The span will be created inside Flush() when we know we'll make an RPC + OtelSpanHandle batch_span; + std::unique_ptr scoped_batch_span; + if (runner.HasOpsToFlush() && OtelTracing::HasActiveContext()) { + batch_span = OtelTracing::StartSpan("pggate.batch"); + if (batch_span.IsActive()) { + std::string session_type_str; + switch (group_session_type) { + case SessionType::kCatalog: session_type_str = "catalog"; break; + case SessionType::kTransactional: session_type_str = "transactional"; break; + case SessionType::kRegular: session_type_str = "regular"; break; + } + batch_span.SetAttribute("yb.session_type", session_type_str); + batch_span.SetAttribute("db.tables", table_names); + batch_span.SetAttribute("yb.catalog_read_count", catalog_read_count); + batch_span.SetAttribute("yb.catalog_write_count", catalog_write_count); + batch_span.SetAttribute("yb.user_read_count", user_read_count); + batch_span.SetAttribute("yb.user_write_count", user_write_count); + scoped_batch_span = std::make_unique(std::move(batch_span)); + } + } + return runner.Flush(std::move(cache_options)); } diff --git a/src/yb/yql/pggate/ybc_pggate.cc b/src/yb/yql/pggate/ybc_pggate.cc index 3b8501741920..fbf6be222b0d 100644 --- a/src/yb/yql/pggate/ybc_pggate.cc +++ b/src/yb/yql/pggate/ybc_pggate.cc @@ -51,6 +51,7 @@ #include "yb/util/curl_util.h" #include "yb/util/flags.h" #include "yb/util/jwt_util.h" +#include "yb/util/otel_tracing.h" #include "yb/util/result.h" #include "yb/util/signal_util.h" #include "yb/util/slice.h" @@ -73,6 +74,12 @@ #include "yb/yql/pggate/util/ybc_util.h" #include "yb/yql/pggate/ybc_pg_typedefs.h" +// Forward declarations for Postgres C functions +extern "C" { + const char* YbGetCurrentTraceparent(void); + void YbClearTraceparent(void); +} + DEFINE_UNKNOWN_int32(pggate_num_connections_to_server, 1, "Number of underlying connections to each server from a PostgreSQL backend process. " "This overrides the value of --num_connections_to_server."); @@ -594,6 +601,186 @@ const YbcPgCallbacks *YBCGetPgCallbacks() { return pgapi->pg_callbacks(); } +void YBCInitOtelTracing(const char* service_name) { + auto status = OtelTracing::InitFromEnv(service_name); + if (!status.ok()) { + LOG(ERROR) << "[OTEL] Failed to initialize OTEL in postgres backend: " << status; + } +} + +const char* YBCGetCurrentTraceparent() { + return YbGetCurrentTraceparent(); +} + +void YBCResetTraceparent() { + YbClearTraceparent(); +} + +// Thread-local storage for OTEL query-level spans +namespace { +thread_local std::unique_ptr g_query_span; +thread_local std::unique_ptr g_parse_span; +thread_local std::unique_ptr g_rewrite_span; +thread_local std::unique_ptr g_plan_span; +thread_local std::vector> g_execute_span_stack; +thread_local std::unique_ptr g_commit_span; +thread_local std::unique_ptr g_abort_span; +} // namespace + +void YBCOtelQueryStart(const char* query_string) { + if (!yb::OtelTracing::IsEnabled()) { + return; + } + + // Get traceparent - it should have been parsed from the query already + const char* traceparent = YbGetCurrentTraceparent(); + + if (!traceparent || traceparent[0] == '\0') { + // No traceparent, no tracing for this query + return; + } + + LOG(INFO) << "[OTEL DEBUG] YBCOtelQueryStart: traceparent='" << traceparent + << "', query='" << (query_string ? query_string : "(null)") << "'"; + + auto span = yb::OtelTracing::StartSpanFromTraceparent("ysql.query", traceparent); + if (span.IsActive()) { + // Set the db.statement attribute with the query string + if (query_string) { + span.SetAttribute("db.statement", std::string(query_string)); + } + span.SetAttribute("db.system", "yugabytedb"); + + g_query_span = std::make_unique(std::move(span)); + LOG(INFO) << "[OTEL DEBUG] YBCOtelQueryStart: Created query span"; + } +} + +void YBCOtelQueryDone() { + if (g_query_span) { + LOG(INFO) << "[OTEL DEBUG] YBCOtelQueryDone: Ending query span"; + g_query_span.reset(); + } +} + +void YBCOtelParseStart() { + if (!g_query_span || !yb::OtelTracing::HasActiveContext()) { + return; + } + + auto span = yb::OtelTracing::StartSpan("ysql.parse"); + if (span.IsActive()) { + g_parse_span = std::make_unique(std::move(span)); + LOG(INFO) << "[OTEL DEBUG] YBCOtelParseStart: Created parse span"; + } +} + +void YBCOtelParseDone() { + if (g_parse_span) { + LOG(INFO) << "[OTEL DEBUG] YBCOtelParseDone: Ending parse span"; + g_parse_span.reset(); + } +} + +void YBCOtelRewriteStart() { + if (!g_query_span || !yb::OtelTracing::HasActiveContext()) { + return; + } + + auto span = yb::OtelTracing::StartSpan("ysql.rewrite"); + if (span.IsActive()) { + g_rewrite_span = std::make_unique(std::move(span)); + LOG(INFO) << "[OTEL DEBUG] YBCOtelRewriteStart: Created rewrite span"; + } +} + +void YBCOtelRewriteDone() { + if (g_rewrite_span) { + LOG(INFO) << "[OTEL DEBUG] YBCOtelRewriteDone: Ending rewrite span"; + g_rewrite_span.reset(); + } +} + +void YBCOtelPlanStart() { + if (!g_query_span || !yb::OtelTracing::HasActiveContext()) { + return; + } + + auto span = yb::OtelTracing::StartSpan("ysql.plan"); + if (span.IsActive()) { + g_plan_span = std::make_unique(std::move(span)); + LOG(INFO) << "[OTEL DEBUG] YBCOtelPlanStart: Created plan span"; + } +} + +void YBCOtelPlanDone() { + if (g_plan_span) { + LOG(INFO) << "[OTEL DEBUG] YBCOtelPlanDone: Ending plan span"; + g_plan_span.reset(); + } +} + +void YBCOtelExecuteStart() { + LOG(INFO) << "[OTEL DEBUG] YBCOtelExecuteStart called: g_query_span=" + << (g_query_span ? "set" : "null") + << ", HasActiveContext=" << yb::OtelTracing::HasActiveContext() + << ", stack_depth=" << g_execute_span_stack.size(); + + if (!g_query_span || !yb::OtelTracing::HasActiveContext()) { + LOG(INFO) << "[OTEL DEBUG] YBCOtelExecuteStart: skipping, condition failed"; + return; + } + + auto span = yb::OtelTracing::StartSpan("ysql.execute"); + if (span.IsActive()) { + g_execute_span_stack.push_back(std::make_unique(std::move(span))); + LOG(INFO) << "[OTEL DEBUG] YBCOtelExecuteStart: Created execute span at depth " + << g_execute_span_stack.size(); + } +} + +void YBCOtelExecuteDone() { + if (!g_execute_span_stack.empty()) { + LOG(INFO) << "[OTEL DEBUG] YBCOtelExecuteDone: Ending execute span at depth " + << g_execute_span_stack.size(); + g_execute_span_stack.pop_back(); + } +} + +void YBCOtelCommitStart() { + if (!g_query_span || !yb::OtelTracing::HasActiveContext()) { + return; + } + + auto span = yb::OtelTracing::StartSpan("ysql.commit"); + if (span.IsActive()) { + g_commit_span = std::make_unique(std::move(span)); + } +} + +void YBCOtelCommitDone() { + if (g_commit_span) { + g_commit_span.reset(); + } +} + +void YBCOtelAbortStart() { + if (!g_query_span || !yb::OtelTracing::HasActiveContext()) { + return; + } + + auto span = yb::OtelTracing::StartSpan("ysql.abort"); + if (span.IsActive()) { + g_abort_span = std::make_unique(std::move(span)); + } +} + +void YBCOtelAbortDone() { + if (g_abort_span) { + g_abort_span.reset(); + } +} + YbcStatus YBCValidateJWT(const char *token, const YbcPgJwtAuthOptions *options) { const std::string token_value(DCHECK_NOTNULL(token)); std::vector identity_claims; diff --git a/src/yb/yql/pggate/ybc_pggate.h b/src/yb/yql/pggate/ybc_pggate.h index fb89eb3b401d..f58a8ccc990a 100644 --- a/src/yb/yql/pggate/ybc_pggate.h +++ b/src/yb/yql/pggate/ybc_pggate.h @@ -119,6 +119,37 @@ const unsigned char* YBCGetLocalTserverUuid(); // Get access to callbacks. const YbcPgCallbacks* YBCGetPgCallbacks(); +// Initialize OpenTelemetry tracing for the current process (e.g., postgres backend). +// Should be called once during process startup. +void YBCInitOtelTracing(const char* service_name); + +// Get the current traceparent string for OTEL tracing (may be empty). +const char* YBCGetCurrentTraceparent(); + +// Clear the current traceparent. +void YBCResetTraceparent(); + +// OTEL Query-level tracing functions (called from DTrace probe points) +// Main query span - extracts traceparent from query_string if present +void YBCOtelQueryStart(const char* query_string); +void YBCOtelQueryDone(void); + +// Query phase child spans +void YBCOtelParseStart(void); +void YBCOtelParseDone(void); +void YBCOtelRewriteStart(void); +void YBCOtelRewriteDone(void); +void YBCOtelPlanStart(void); +void YBCOtelPlanDone(void); +void YBCOtelExecuteStart(void); +void YBCOtelExecuteDone(void); + +// Transaction phase spans +void YBCOtelCommitStart(void); +void YBCOtelCommitDone(void); +void YBCOtelAbortStart(void); +void YBCOtelAbortDone(void); + int64_t YBCGetPgggateCurrentAllocatedBytes(); int64_t YBCGetActualHeapSizeBytes();