CNDB-15508: Query planner metrics

This commit adds new metrics related to the operation of SAI query
planner. The metrics should help checking if the query planner makes
proper decisions by correlating them with the other metrics,
e.g. the metrics of the actual query execution.

Per-query metrics (histograms):
- `RowsEstimated`: the estimated number of rows to be returned by
   the query
- `CostEstimated`: the abstract cost of query execution
- `InverseSelectivityEstimated`: the inverse of query selectivity,
  before applying the query LIMIT
  (1 means the query selects all rows, 10 means it
  selects every 10th row, etc.)
- `IndexReferencesInQuery`: the number of index references in the
  unoptimized query execution plan (the same index may
  be referenced multiple times and counts separately)
- `IndexReferencesInPlan`: the number of index references in the
  optimized query execution plan (the same index may
  be referenced multiple times and counts separately)

Per-table:
- `TotalRowsEstimated`: counts the sum of all row estimates from
  all completed queries
- `TotalCostEstimated`: counts the sum of all cost estimates from
  all completed queries
- `TotalQueriesCompletedInSelectivityGroup{N}`, where N in [0, 12):
  counts the number of completed queries with selectivity S:
  10^(-N-1) < S <= 10^(-N) for N < 11,
              S <= 10^(-N) for N = 11
  In other words, the higher the group N, the smaller fraction of rows
  the query is estimated to return. The selectivity calculation
  does not include the final LIMIT of the query.

Author: Piotr Kołaczkowski

src/java/org/apache/cassandra/index/sai/QueryContext.java

@@ -25,6 +25,7 @@
 import com.google.common.annotations.VisibleForTesting;
 
 import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.index.sai.plan.Plan;
 import org.apache.cassandra.index.sai.utils.AbortedOperationException;
 import org.apache.cassandra.utils.MonotonicClock;
 
@@ -66,9 +67,10 @@ public class QueryContext
 
     private final LongAdder shadowedPrimaryKeyCount = new LongAdder();
 
-    // Determines the order of using indexes for filtering and sorting.
-    // Null means the query execution order hasn't been decided yet.
-    private FilterSortOrder filterSortOrder = null;
+    private Plan originalPlan = null;
+
+
+    private Plan optimizedPlan = null;
 
     @VisibleForTesting
     public QueryContext()
@@ -145,11 +147,6 @@ public void addAnnGraphSearchLatency(long val)
         annGraphSearchLatency.add(val);
     }
 
-    public void setFilterSortOrder(FilterSortOrder filterSortOrder)
-    {
-        this.filterSortOrder = filterSortOrder;
-    }
-
     // getters
 
     public long sstablesHit()
@@ -208,9 +205,14 @@ public long annGraphSearchLatency()
         return annGraphSearchLatency.longValue();
     }
 
-    public FilterSortOrder filterSortOrder()
+    public Plan optimizedPlan()
     {
-        return filterSortOrder;
+        return optimizedPlan;
+    }
+
+    public Plan originalPlan()
+    {
+        return originalPlan;
     }
 
     public void checkpoint()
@@ -246,17 +248,14 @@ public void updateAnnRerankFloor(float observedFloor)
             annRerankFloor = max(annRerankFloor, observedFloor);
     }
 
-    /**
-     * Determines the order of filtering and sorting operations.
-     * Currently used only by vector search.
-     */
-    public enum FilterSortOrder
+    public void setOriginalPlan(Plan originalPlan)
     {
-        /** First get the matching keys from the non-vector indexes, then use vector index to return the top K by similarity order */
-        SEARCH_THEN_ORDER,
+        this.originalPlan = originalPlan;
+    }
 
-        /** First get the candidates in ANN order from the vector index, then fetch the rows and filter them until we find K matching the predicates */
-        SCAN_THEN_FILTER
+    public void setOptimizedPlan(Plan optimizedPlan)
+    {
+        this.optimizedPlan = optimizedPlan;
     }
 
     public Snapshot snapshot()
@@ -290,7 +289,18 @@ public static class Snapshot
         public final long queryTimeouts;
         public final long annGraphSearchLatency;
         public final long shadowedPrimaryKeyCount;
-        public final FilterSortOrder filterSortOrder;
+
+        public final boolean searchExecutedBeforeOrder;
+        public final boolean filterExecutedAfterOrderedScan;
+
+        public final double rowsEstimated;
+        public final double selectivityEstimated;
+        public final double costEstimated;
+
+        public final int indexReferencesInQuery;
+        public final int indexReferencesInPlan;
+
+
 
         /**
          * Creates a snapshot of all the metrics in the given {@link QueryContext}.
@@ -315,7 +325,16 @@ private Snapshot(QueryContext context)
             queryTimeouts = context.queryTimeouts();
             annGraphSearchLatency = context.annGraphSearchLatency();
             shadowedPrimaryKeyCount = context.getShadowedPrimaryKeyCount();
-            filterSortOrder = context.filterSortOrder();
+
+            costEstimated = context.optimizedPlan().fullCost();
+            rowsEstimated = context.optimizedPlan().expectedRows();
+            selectivityEstimated = context.optimizedPlan().selectivity();
+            indexReferencesInQuery = context.originalPlan().referencedIndexCount();
+            indexReferencesInPlan = context.optimizedPlan().referencedIndexCount();
+            searchExecutedBeforeOrder = context.optimizedPlan().isSearchThenOrderHybrid();
+            filterExecutedAfterOrderedScan = context.optimizedPlan().isOrderedScanThenFilterHybrid();
         }
     }
+
+
 }

src/java/org/apache/cassandra/index/sai/metrics/TableQueryMetrics.java

@@ -115,7 +115,7 @@ public void record(QueryContext context, ReadCommand command)
         {
             final long queryLatencyMicros = TimeUnit.NANOSECONDS.toMicros(snapshot.totalQueryTimeNs);
 
-            if (snapshot.filterSortOrder == QueryContext.FilterSortOrder.SEARCH_THEN_ORDER)
+            if (snapshot.searchExecutedBeforeOrder)
             {
                 Tracing.trace("Index query accessed memtable indexes, {}, and {}, selected {} before ranking, " +
                               "post-filtered {} in {}, and took {} microseconds.",
@@ -189,11 +189,18 @@ public static class PerTable extends AbstractQueryMetrics
     {
         public static final String METRIC_TYPE = "TableQueryMetrics";
 
+        // How small selectivities we track, queries with selectivity < 10^(-12) will all go to one group
+        private static final int SELECTIVITY_GROUPS_COUNT = 12;
+
         public final Counter totalQueryTimeouts;
         public final Counter totalPartitionReads;
         public final Counter totalRowsFiltered;
         public final Counter totalQueriesCompleted;
 
+        public final Counter totalRowsEstimated;
+        public final Counter totalCostEstimated;
+        public final Counter[] totalQueriesCompletedInSelectivityGroup;
+
         public final Counter sortThenFilterQueriesCompleted;
         public final Counter filterThenSortQueriesCompleted;
 
@@ -211,6 +218,13 @@ public PerTable(TableMetadata table, QueryKind queryKind, Predicate<ReadCommand>
             totalQueriesCompleted = Metrics.counter(createMetricName("TotalQueriesCompleted"));
             totalQueryTimeouts = Metrics.counter(createMetricName("TotalQueryTimeouts"));
 
+            totalRowsEstimated = Metrics.counter(createMetricName("TotalRowsEstimated"));
+            totalCostEstimated = Metrics.counter(createMetricName("TotalCostEstimated"));
+
+            totalQueriesCompletedInSelectivityGroup = new Counter[SELECTIVITY_GROUPS_COUNT];
+            for (int i = 0; i < totalQueriesCompletedInSelectivityGroup.length; i++)
+                totalQueriesCompletedInSelectivityGroup[i] = Metrics.counter(createMetricName("TotalQueriesCompletedInSelectivityGroup" + i));
+
             sortThenFilterQueriesCompleted = Metrics.counter(createMetricName("SortThenFilterQueriesCompleted"));
             filterThenSortQueriesCompleted = Metrics.counter(createMetricName("FilterThenSortQueriesCompleted"));
         }
@@ -227,10 +241,16 @@ public void record(QueryContext.Snapshot snapshot)
             totalQueriesCompleted.inc();
             totalPartitionReads.inc(snapshot.partitionsRead);
             totalRowsFiltered.inc(snapshot.rowsFiltered);
+            totalCostEstimated.inc(Math.round(snapshot.costEstimated));
+            totalRowsEstimated.inc(Math.round(snapshot.rowsEstimated));
 
-            if (snapshot.filterSortOrder == QueryContext.FilterSortOrder.SCAN_THEN_FILTER)
+            int selectivityGroup = (int)(Math.min(totalQueriesCompletedInSelectivityGroup.length - 1,
+                                                  Math.floor(-Math.log10(snapshot.selectivityEstimated))));
+            totalQueriesCompletedInSelectivityGroup[selectivityGroup].inc();
+
+            if (snapshot.filterExecutedAfterOrderedScan)
                 sortThenFilterQueriesCompleted.inc();
-            else if (snapshot.filterSortOrder == QueryContext.FilterSortOrder.SEARCH_THEN_ORDER)
+            if (snapshot.searchExecutedBeforeOrder)
                 filterThenSortQueriesCompleted.inc();
         }
     }
@@ -276,6 +296,32 @@ public static class PerQuery extends AbstractQueryMetrics
          */
         public final Timer annGraphSearchLatency;
 
+        /** Query execution cost as estimated by the planner */
+        public final Histogram costEstimated;
+
+        /** Number of rows returned by the query estimated by the planner */
+        public final Histogram rowsEstimated;
+
+        /**
+         * Inverse selectivity of the query, before applying the LIMIT clause.
+         * Denotes by how many rows on average the index iterator
+         * is advanced to get the next matching key. High inverse selectivity means many rows
+         * are skipped thanks to use of the index. Inverse selectivity equals 1 means rows are not skipped.
+         * We track inverse of selectivity, because histograms do not support fractional values,
+         * and selectivity is <= 1.0.
+         */
+        public final Histogram inverseSelectivityEstimated;
+
+        /** 
+         * Number of indexes referenced by the optimized query plan.
+         * The same index referenced from unrelated query clauses, 
+         * leading to separate index searches, are counted separately.  
+         */
+        public final Histogram indexReferencesInPlan;
+
+        /** Number of indexes referenced by the original query plan before optimization (as stated in the query text) */
+        public final Histogram indexReferencesInQuery;
+        
         /**
          * @param table the table to measure metrics for
          * @param queryKind an identifier for the kind of query which metrics are being recorded for
@@ -304,6 +350,12 @@ public PerQuery(TableMetadata table, QueryKind queryKind, Predicate<ReadCommand>
 
             // Key vector metrics that translate to performance
             annGraphSearchLatency = Metrics.timer(createMetricName("ANNGraphSearchLatency"));
+
+            costEstimated = Metrics.histogram(createMetricName("CostEstimated"), false);
+            rowsEstimated = Metrics.histogram(createMetricName("RowsEstimated"), true);
+            inverseSelectivityEstimated = Metrics.histogram(createMetricName("InverseSelectivityEstimated"), false);
+            indexReferencesInPlan = Metrics.histogram(createMetricName("IndexReferencesInPlan"), true);
+            indexReferencesInQuery = Metrics.histogram(createMetricName("IndexReferencesInQuery"), false);
         }
 
         @Override
@@ -340,6 +392,13 @@ public void record(QueryContext.Snapshot snapshot)
             }
 
             shadowedKeysScannedHistogram.update(snapshot.shadowedPrimaryKeyCount);
+
+            costEstimated.update(Math.round(snapshot.costEstimated));
+            rowsEstimated.update(Math.round(snapshot.rowsEstimated));
+            inverseSelectivityEstimated.update(Math.round(1.0 / snapshot.selectivityEstimated));
+
+            indexReferencesInQuery.update(snapshot.indexReferencesInQuery);
+            indexReferencesInPlan.update(snapshot.indexReferencesInPlan);
         }
     }
 }