getting trace output

bin/phoenix_sandbox.py

@@ -26,6 +26,7 @@
 import phoenix_utils
 
 phoenix_utils.setPath()
+phoenix_utils.set_tracing()
 
 base_dir = os.path.join(phoenix_utils.current_dir, '..')
 phoenix_target_dir = os.path.join(base_dir, 'phoenix-core', 'target')
@@ -44,9 +45,10 @@
 with open(cp_file_path, 'r') as cp_file:
     cp_components.append(cp_file.read())
 
-java_cmd = ("java $PHOENIX_OPTS -Dlog4j2.configurationFile=file:%s " +
-                "-cp %s org.apache.phoenix.Sandbox") % (
-                            logging_config, ":".join(cp_components))
+java_cmd = ("java -javaagent:%s -Dlog4j2.configurationFile=file:%s " +
+            ' ' + phoenix_utils.phoenix_trace_opts + ' ' +
+                "-cp %s org.apache.phoenix.Sandbox") % ( 
+                    phoenix_utils.opentelemetry_agent_jar, logging_config, ":".join(cp_components))
 
 proc = subprocess.Popen(java_cmd, shell=True)
 try:

bin/phoenix_utils.py

@@ -86,11 +86,12 @@ def setPath():
     LOGGING_JAR_PATTERN2 = "log4j-api*.jar"
     LOGGING_JAR_PATTERN3 = "log4j-1.2-api*.jar"
     SQLLINE_WITH_DEPS_PATTERN = "sqlline-*-jar-with-dependencies.jar"
-
+    OPENTELEMETRY_AGENT_PATTERN = "opentelemetry-javaagent*.jar"
 
     OVERRIDE_SLF4J_BACKEND = "OVERRIDE_SLF4J_BACKEND_JAR_LOCATION"
     OVERRIDE_LOGGING = "OVERRIDE_LOGGING_JAR_LOCATION"
     OVERRIDE_SQLLINE = "OVERRIDE_SQLLINE_JAR_LOCATION"
+    OVERRIDE_OPENTELEMETRY_AGENT = "OVERRIDE_OPENTELEMETRY_AGENT_JAR_LOCATION"
 
     # Backward support old env variable PHOENIX_LIB_DIR replaced by PHOENIX_CLASS_PATH
     global phoenix_class_path
@@ -189,6 +190,18 @@ def setPath():
         logging_jar += ":"+findFileInPathWithoutRecursion(LOGGING_JAR_PATTERN2, os.path.join(current_dir, "..","lib"))
         logging_jar += ":"+findFileInPathWithoutRecursion(LOGGING_JAR_PATTERN3, os.path.join(current_dir, "..","lib"))
 
+    global opentelemetry_agent_jar
+    opentelemetry_agent_jar = os.environ.get(OVERRIDE_OPENTELEMETRY_AGENT)
+    if opentelemetry_agent_jar is None or opentelemetry_agent_jar == "":
+        opentelemetry_agent_jar = findFileInPathWithoutRecursion(OPENTELEMETRY_AGENT_PATTERN, os.path.join(current_dir, "..","lib/tracing"))
+
+    return ""
+
+def set_tracing():
+    global phoenix_trace_opts
+    phoenix_trace_opts = os.environ.get("PHOENIX_TRACE_OPTS")
+    if phoenix_trace_opts is None or phoenix_trace_opts == "":
+        phoenix_trace_opts = "-Dotel.traces.exporter=logging -Dotel.metrics.exporter=none"
     return ""
 
 def shell_quote(args):
@@ -226,3 +239,4 @@ def common_sqlline_args(parser):
     print("sqlline_with_deps_jar:", sqlline_with_deps_jar)
     print("slf4j_backend_jar:", slf4j_backend_jar)
     print("logging_jar:", logging_jar)
+    print("opentelemetry_agent_jar:", opentelemetry_agent_jar)

bin/sqlline.py

@@ -46,6 +46,7 @@ def kill_child():
 atexit.register(kill_child)
 
 phoenix_utils.setPath()
+phoenix_utils.set_tracing()
 
 parser = argparse.ArgumentParser(description='Launches the Apache Phoenix Client.')
 # Positional argument 'zookeepers' is optional. The PhoenixDriver will automatically populate
@@ -118,14 +119,15 @@ def kill_child():
 else:
     disable_jna = ""
 
-java_cmd = java + ' $PHOENIX_OPTS ' + \
+java_cmd = java + ' -javaagent:' + phoenix_utils.opentelemetry_agent_jar + ' $PHOENIX_OPTS ' + \
     ' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + \
     phoenix_utils.hadoop_conf + os.pathsep + \
     phoenix_utils.sqlline_with_deps_jar + os.pathsep + \
     phoenix_utils.slf4j_backend_jar + os.pathsep + \
     phoenix_utils.logging_jar + os.pathsep + \
     phoenix_utils.phoenix_client_embedded_jar + \
     '" -Dlog4j2.configurationFile=file:' + os.path.join(phoenix_utils.current_dir, "log4j2.properties") + \
+    ' ' + phoenix_utils.phoenix_trace_opts + ' ' + \
     disable_jna + \
     " sqlline.SqlLine -d org.apache.phoenix.jdbc.PhoenixDriver" + \
     (not args.noconnect and " -u jdbc:phoenix:" + phoenix_utils.shell_quote([zookeeper]) or "") + \

phoenix-client-parent/pom.xml

@@ -184,6 +184,8 @@
                 <pattern>io/</pattern>
                 <shadedPattern>${shaded.package}.io.</shadedPattern>
                 <excludes>
+                  <!-- Must interoperate with opentelemetry-javaagent -->
+                  <exclude>io.opentelemetry/**</exclude>
                   <!-- Exclude config keys for Hadoop that look like package names -->
                   <exclude>io/compression/**</exclude>
                   <exclude>io/mapfile/**</exclude>

phoenix-core/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java

@@ -365,10 +365,7 @@ public final ResultIterator iterator(final Map<ImmutableBytesPtr,ServerCache> ca
                     "Iterator for table " + context.getCurrentTable().getTable().getName() + " ready: " + iterator, connection));
         }
 
-        //FIXME where to set the root span attributes ?
-        //FIXME add the plan as an attribute, to avoid cost when not recording
-        Span span = TraceUtil.createSpan("Creating basic query for " + getPlanSteps(iterator));
-        return new TracingIterator(span, iterator);
+        return new TracingIterator(iterator);
     }
 
     private void serializeIndexMaintainerIntoScan(Scan scan, PTable dataTable) throws SQLException {
@@ -520,12 +517,6 @@ public ExplainPlan getExplainPlan() throws SQLException {
         return explainPlan;
     }
 
-    private List<String> getPlanSteps(ResultIterator iterator) {
-        List<String> planSteps = Lists.newArrayListWithExpectedSize(5);
-        iterator.explain(planSteps);
-        return planSteps;
-    }
-
     private Pair<List<String>, ExplainPlanAttributes> getPlanStepsV2(
             ResultIterator iterator) {
         List<String> planSteps = Lists.newArrayListWithExpectedSize(5);

phoenix-core/src/main/java/org/apache/phoenix/trace/TraceUtil.java

@@ -24,17 +24,16 @@
 import java.util.function.BiConsumer;
 import java.util.function.Supplier;
 
-import org.apache.phoenix.call.CallRunner;
 import org.apache.phoenix.thirdparty.com.google.common.base.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import io.opentelemetry.api.GlobalOpenTelemetry;
-import io.opentelemetry.context.Context;
 import io.opentelemetry.api.trace.Span;
 import io.opentelemetry.api.trace.SpanKind;
 import io.opentelemetry.api.trace.StatusCode;
 import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.context.Context;
 import io.opentelemetry.context.Scope;
 
 // This is copied from org.apache.hadoop.hbase.trace.TraceUtil

phoenix-core/src/main/java/org/apache/phoenix/trace/TracingIterator.java

@@ -18,10 +18,12 @@
 package org.apache.phoenix.trace;
 
 import java.sql.SQLException;
+import java.util.List;
 
 import org.apache.phoenix.iterate.DelegateResultIterator;
 import org.apache.phoenix.iterate.ResultIterator;
 import org.apache.phoenix.schema.tuple.Tuple;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
 
 import io.opentelemetry.api.trace.Span;
 import io.opentelemetry.api.trace.StatusCode;
@@ -32,24 +34,34 @@
  */
 public class TracingIterator extends DelegateResultIterator {
 
-    private Span span;
-    private Scope scope;
+    private final Span span;
+    private final Scope scope;
     private boolean started;
 
     /**
      * @param scope a scope with a non-null span
      * @param iterator delegate
      */
-    public TracingIterator(Span Span, ResultIterator iterator) {
+    public TracingIterator(ResultIterator iterator) {
         super(iterator);
-        this.span = Span;
+        //FIXME where to set the root span attributes ?
+        //FIXME add the plan as an attribute, to avoid cost when not recording
+        span = TraceUtil.createSpan("Creating basic query for " + getPlanSteps(iterator));
         scope = span.makeCurrent();
     }
 
+    private List<String> getPlanSteps(ResultIterator iterator) {
+        List<String> planSteps = Lists.newArrayListWithExpectedSize(5);
+        iterator.explain(planSteps);
+        return planSteps;
+    }
+
     @Override
     public void close() throws SQLException {
         scope.close();
         span.end();
+        // We start spans in the parent first, so we must close it last.
+        // Even if it is unlikely that we'd ever add tracing to DelegateResultIterator.
         super.close();
     }
 

phoenix-server/pom.xml

@@ -121,6 +121,8 @@
                         <exclude>org.slf4j:*</exclude>
                         <exclude>org.apache.logging.log4j:*</exclude>
                         <exclude>org.apache.hbase.thirdparty:*</exclude>
+                        <!-- This would break support for Hbase 2.4 
+                        <exclude>io.opentelemetry:*</exclude> -->
                     </excludes>
                 </artifactSet>
                 <filters>