CNDB-15508: Query planner metrics

This commit adds new metrics related to the operation of SAI query
planner. The metrics should help checking if the query planner makes
proper decisions by correlating them with the other metrics,
e.g. the metrics of the actual query execution.

Per-query metrics (histograms):
- `RowsEstimated`: the estimated number of rows to be returned by
   the query
- `CostEstimated`: the abstract cost of query execution
- `LogSelectivityEstimated`: minus decimal logarithm of query
   selectivity,  before applying the query LIMIT (0 means the query
   selects all rows, 5 means it selects 10^(-5) = 0.00001 subset
   of rows)
- `IndexReferencesInQuery`: the number of index references in the
  unoptimized query execution plan (the same index may
  be referenced multiple times and counts separately)
- `IndexReferencesInPlan`: the number of index references in the
  optimized query execution plan (the same index may
  be referenced multiple times and counts separately)

Per-table:
- `TotalRowsEstimated`: counts the sum of all row estimates from
  all completed queries
- `TotalCostEstimated`: counts the sum of all cost estimates from
  all completed queries

Author: Piotr Kołaczkowski

src/java/org/apache/cassandra/index/sai/QueryContext.java

@@ -20,11 +20,14 @@
 
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.LongAdder;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 import javax.annotation.concurrent.NotThreadSafe;
 
 import com.google.common.annotations.VisibleForTesting;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.index.sai.plan.Plan;
 import org.apache.cassandra.index.sai.utils.AbortedOperationException;
 import org.apache.cassandra.utils.MonotonicClock;
 
@@ -66,9 +69,10 @@ public class QueryContext
 
     private final LongAdder shadowedPrimaryKeyCount = new LongAdder();
 
-    // Determines the order of using indexes for filtering and sorting.
-    // Null means the query execution order hasn't been decided yet.
-    private FilterSortOrder filterSortOrder = null;
+    private Plan originalPlan = null;
+
+
+    private Plan optimizedPlan = null;
 
     @VisibleForTesting
     public QueryContext()
@@ -145,11 +149,6 @@ public void addAnnGraphSearchLatency(long val)
         annGraphSearchLatency.add(val);
     }
 
-    public void setFilterSortOrder(FilterSortOrder filterSortOrder)
-    {
-        this.filterSortOrder = filterSortOrder;
-    }
-
     // getters
 
     public long sstablesHit()
@@ -208,9 +207,18 @@ public long annGraphSearchLatency()
         return annGraphSearchLatency.longValue();
     }
 
-    public FilterSortOrder filterSortOrder()
+    /** Can return null when query plan hasn't been prepared and optimized yet */
+    @Nullable
+    public Plan optimizedPlan()
     {
-        return filterSortOrder;
+        return optimizedPlan;
+    }
+
+    /** Can return null when query plan hasn't been prepared yet */
+    @Nullable
+    public Plan originalPlan()
+    {
+        return originalPlan;
     }
 
     public void checkpoint()
@@ -246,17 +254,14 @@ public void updateAnnRerankFloor(float observedFloor)
             annRerankFloor = max(annRerankFloor, observedFloor);
     }
 
-    /**
-     * Determines the order of filtering and sorting operations.
-     * Currently used only by vector search.
-     */
-    public enum FilterSortOrder
+    public void setOriginalPlan(Plan originalPlan)
     {
-        /** First get the matching keys from the non-vector indexes, then use vector index to return the top K by similarity order */
-        SEARCH_THEN_ORDER,
+        this.originalPlan = originalPlan;
+    }
 
-        /** First get the candidates in ANN order from the vector index, then fetch the rows and filter them until we find K matching the predicates */
-        SCAN_THEN_FILTER
+    public void setOptimizedPlan(Plan optimizedPlan)
+    {
+        this.optimizedPlan = optimizedPlan;
     }
 
     public Snapshot snapshot()
@@ -290,7 +295,9 @@ public static class Snapshot
         public final long queryTimeouts;
         public final long annGraphSearchLatency;
         public final long shadowedPrimaryKeyCount;
-        public final FilterSortOrder filterSortOrder;
+
+        @Nullable
+        public final QueryPlanInfo queryPlanInfo;
 
         /**
          * Creates a snapshot of all the metrics in the given {@link QueryContext}.
@@ -315,7 +322,40 @@ private Snapshot(QueryContext context)
             queryTimeouts = context.queryTimeouts();
             annGraphSearchLatency = context.annGraphSearchLatency();
             shadowedPrimaryKeyCount = context.getShadowedPrimaryKeyCount();
-            filterSortOrder = context.filterSortOrder();
+
+            Plan originalPlan = context.originalPlan();
+            Plan optimizedPlan = context.optimizedPlan();
+            if (originalPlan != null && optimizedPlan != null) {
+                queryPlanInfo = new QueryPlanInfo(originalPlan, optimizedPlan);
+            } else {
+                queryPlanInfo = null;
+            }
         }
+
+        public static class QueryPlanInfo
+        {
+            public final boolean searchExecutedBeforeOrder;
+            public final boolean filterExecutedAfterOrderedScan;
+
+            public final double rowsEstimated;
+            public final double selectivityEstimated;
+            public final double costEstimated;
+
+            public final int indexReferencesInQuery;
+            public final int indexReferencesInPlan;
+
+            public QueryPlanInfo(@Nonnull Plan originalPlan, @Nonnull Plan optimizedPlan)
+            {
+                this.costEstimated = optimizedPlan.fullCost();
+                this.rowsEstimated = optimizedPlan.expectedRows();
+                this.selectivityEstimated = optimizedPlan.selectivity();
+                this.indexReferencesInQuery = originalPlan.referencedIndexCount();
+                this.indexReferencesInPlan = optimizedPlan.referencedIndexCount();
+                this.searchExecutedBeforeOrder = optimizedPlan.isSearchThenOrderHybrid();
+                this.filterExecutedAfterOrderedScan = optimizedPlan.isOrderedScanThenFilterHybrid();
+            }
+        }
+
     }
+
 }

src/java/org/apache/cassandra/index/sai/metrics/TableQueryMetrics.java

@@ -115,7 +115,7 @@ public void record(QueryContext context, ReadCommand command)
         {
             final long queryLatencyMicros = TimeUnit.NANOSECONDS.toMicros(snapshot.totalQueryTimeNs);
 
-            if (snapshot.filterSortOrder == QueryContext.FilterSortOrder.SEARCH_THEN_ORDER)
+            if (snapshot.queryPlanInfo != null && snapshot.queryPlanInfo.searchExecutedBeforeOrder)
             {
                 Tracing.trace("Index query accessed memtable indexes, {}, and {}, selected {} before ranking, " +
                               "post-filtered {} in {}, and took {} microseconds.",
@@ -194,6 +194,9 @@ public static class PerTable extends AbstractQueryMetrics
         public final Counter totalRowsFiltered;
         public final Counter totalQueriesCompleted;
 
+        public final Counter totalRowsEstimated;
+        public final Counter totalCostEstimated;
+
         public final Counter sortThenFilterQueriesCompleted;
         public final Counter filterThenSortQueriesCompleted;
 
@@ -210,6 +213,8 @@ public PerTable(TableMetadata table, QueryKind queryKind, Predicate<ReadCommand>
             totalRowsFiltered = Metrics.counter(createMetricName("TotalRowsFiltered"));
             totalQueriesCompleted = Metrics.counter(createMetricName("TotalQueriesCompleted"));
             totalQueryTimeouts = Metrics.counter(createMetricName("TotalQueryTimeouts"));
+            totalRowsEstimated = Metrics.counter(createMetricName("TotalRowsEstimated"));
+            totalCostEstimated = Metrics.counter(createMetricName("TotalCostEstimated"));
 
             sortThenFilterQueriesCompleted = Metrics.counter(createMetricName("SortThenFilterQueriesCompleted"));
             filterThenSortQueriesCompleted = Metrics.counter(createMetricName("FilterThenSortQueriesCompleted"));
@@ -228,10 +233,17 @@ public void record(QueryContext.Snapshot snapshot)
             totalPartitionReads.inc(snapshot.partitionsRead);
             totalRowsFiltered.inc(snapshot.rowsFiltered);
 
-            if (snapshot.filterSortOrder == QueryContext.FilterSortOrder.SCAN_THEN_FILTER)
-                sortThenFilterQueriesCompleted.inc();
-            else if (snapshot.filterSortOrder == QueryContext.FilterSortOrder.SEARCH_THEN_ORDER)
-                filterThenSortQueriesCompleted.inc();
+            QueryContext.Snapshot.QueryPlanInfo queryPlanInfo = snapshot.queryPlanInfo;
+            if (queryPlanInfo != null)
+            {
+                totalCostEstimated.inc(Math.round(queryPlanInfo.costEstimated));
+                totalRowsEstimated.inc(Math.round(queryPlanInfo.rowsEstimated));
+
+                if (queryPlanInfo.filterExecutedAfterOrderedScan)
+                    sortThenFilterQueriesCompleted.inc();
+                if (queryPlanInfo.searchExecutedBeforeOrder)
+                    filterThenSortQueriesCompleted.inc();
+            }
         }
     }
 
@@ -276,6 +288,28 @@ public static class PerQuery extends AbstractQueryMetrics
          */
         public final Timer annGraphSearchLatency;
 
+        /** Query execution cost as estimated by the planner */
+        public final Histogram costEstimated;
+
+        /** Number of rows returned by the query estimated by the planner */
+        public final Histogram rowsEstimated;
+
+        /**
+         * Negative deceimal logarithm of selectivity of the query, before applying the LIMIT clause.
+         * We use logarithm because selectivity values can be very small (e.g. 10^-9).
+         */
+        public final Histogram logSelectivityEstimated;
+
+        /** 
+         * Number of indexes referenced by the optimized query plan.
+         * The same index referenced from unrelated query clauses, 
+         * leading to separate index searches, are counted separately.  
+         */
+        public final Histogram indexReferencesInPlan;
+
+        /** Number of indexes referenced by the original query plan before optimization (as stated in the query text) */
+        public final Histogram indexReferencesInQuery;
+        
         /**
          * @param table the table to measure metrics for
          * @param queryKind an identifier for the kind of query which metrics are being recorded for
@@ -304,6 +338,12 @@ public PerQuery(TableMetadata table, QueryKind queryKind, Predicate<ReadCommand>
 
             // Key vector metrics that translate to performance
             annGraphSearchLatency = Metrics.timer(createMetricName("ANNGraphSearchLatency"));
+
+            costEstimated = Metrics.histogram(createMetricName("CostEstimated"), false);
+            rowsEstimated = Metrics.histogram(createMetricName("RowsEstimated"), true);
+            logSelectivityEstimated = Metrics.histogram(createMetricName("LogSelectivityEstimated"), true);
+            indexReferencesInPlan = Metrics.histogram(createMetricName("IndexReferencesInPlan"), true);
+            indexReferencesInQuery = Metrics.histogram(createMetricName("IndexReferencesInQuery"), false);
         }
 
         @Override
@@ -340,6 +380,17 @@ public void record(QueryContext.Snapshot snapshot)
             }
 
             shadowedKeysScannedHistogram.update(snapshot.shadowedPrimaryKeyCount);
+
+            QueryContext.Snapshot.QueryPlanInfo queryPlanInfo = snapshot.queryPlanInfo;
+            if (queryPlanInfo != null)
+            {
+                costEstimated.update(Math.round(queryPlanInfo.costEstimated));
+                rowsEstimated.update(Math.round(queryPlanInfo.rowsEstimated));
+                double logSelectivity = -Math.log10(queryPlanInfo.selectivityEstimated);
+                logSelectivityEstimated.update((int) (Math.min(20, Math.floor(logSelectivity))));
+                indexReferencesInQuery.update(queryPlanInfo.indexReferencesInQuery);
+                indexReferencesInPlan.update(queryPlanInfo.indexReferencesInPlan);
+            }
         }
     }
 }