Support typeOfMetrics query (apache#4686)

* Add data type to the column definition to replace the simple isValue definition. And top N query is also requiring value column now.

* Support typeOfMetrics query.

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java

@@ -32,6 +32,7 @@
 import org.apache.skywalking.oap.server.core.query.AlarmQueryService;
 import org.apache.skywalking.oap.server.core.query.LogQueryService;
 import org.apache.skywalking.oap.server.core.query.MetadataQueryService;
+import org.apache.skywalking.oap.server.core.query.MetricsMetadataQueryService;
 import org.apache.skywalking.oap.server.core.query.MetricsQueryService;
 import org.apache.skywalking.oap.server.core.query.ProfileTaskQueryService;
 import org.apache.skywalking.oap.server.core.query.TopNRecordsQueryService;
@@ -90,6 +91,7 @@ private void addProfileService(List<Class> classes) {
 
     private void addQueryService(List<Class> classes) {
         classes.add(TopologyQueryService.class);
+        classes.add(MetricsMetadataQueryService.class);
         classes.add(MetricsQueryService.class);
         classes.add(TraceQueryService.class);
         classes.add(LogQueryService.class);

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java

@@ -48,6 +48,7 @@
 import org.apache.skywalking.oap.server.core.query.AlarmQueryService;
 import org.apache.skywalking.oap.server.core.query.LogQueryService;
 import org.apache.skywalking.oap.server.core.query.MetadataQueryService;
+import org.apache.skywalking.oap.server.core.query.MetricsMetadataQueryService;
 import org.apache.skywalking.oap.server.core.query.MetricsQueryService;
 import org.apache.skywalking.oap.server.core.query.ProfileTaskQueryService;
 import org.apache.skywalking.oap.server.core.query.TopNRecordsQueryService;
@@ -213,6 +214,7 @@ public void prepare() throws ServiceNotProvidedException, ModuleStartException {
             NetworkAddressAliasCache.class, new NetworkAddressAliasCache(moduleConfig));
 
         this.registerServiceImplementation(TopologyQueryService.class, new TopologyQueryService(getManager()));
+        this.registerServiceImplementation(MetricsMetadataQueryService.class, new MetricsMetadataQueryService());
         this.registerServiceImplementation(MetricsQueryService.class, new MetricsQueryService(getManager()));
         this.registerServiceImplementation(TraceQueryService.class, new TraceQueryService(getManager()));
         this.registerServiceImplementation(LogQueryService.class, new LogQueryService(getManager()));

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/ApdexMetrics.java

@@ -60,7 +60,7 @@ public abstract class ApdexMetrics extends Metrics implements IntValueHolder {
     private int tNum;
     @Getter
     @Setter
-    @Column(columnName = VALUE, isValue = true, function = Function.Avg)
+    @Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE, function = Function.Avg)
     private int value;
 
     @Entrance

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/CPMMetrics.java

@@ -34,7 +34,7 @@ public abstract class CPMMetrics extends Metrics implements LongValueHolder {
 
     @Getter
     @Setter
-    @Column(columnName = VALUE, isValue = true, function = Function.Avg)
+    @Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE, function = Function.Avg)
     private long value;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/CountMetrics.java

@@ -33,7 +33,7 @@ public abstract class CountMetrics extends Metrics implements LongValueHolder {
 
     @Getter
     @Setter
-    @Column(columnName = VALUE, isValue = true, function = Function.Sum)
+    @Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE, function = Function.Sum)
     private long value;
 
     @Entrance

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/DoubleAvgMetrics.java

@@ -44,7 +44,7 @@ public abstract class DoubleAvgMetrics extends Metrics implements DoubleValueHol
     private long count;
     @Getter
     @Setter
-    @Column(columnName = VALUE, isValue = true, function = Function.Avg)
+    @Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE, function = Function.Avg)
     private double value;
 
     @Entrance

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/HistogramMetrics.java

@@ -39,13 +39,9 @@ public abstract class HistogramMetrics extends Metrics {
 
     public static final String DATASET = "dataset";
 
-    /**
-     * The special case when the column is isValue = true, but storageOnly = true, because it is {@link DataTable} type,
-     * this column can't be query by the aggregation way.
-     */
     @Getter
     @Setter
-    @Column(columnName = DATASET, isValue = true, storageOnly = true)
+    @Column(columnName = DATASET, dataType = Column.ValueDataType.HISTOGRAM, storageOnly = true)
     private DataTable dataset = new DataTable(30);
 
     /**

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/LongAvgMetrics.java

@@ -44,7 +44,7 @@ public abstract class LongAvgMetrics extends Metrics implements LongValueHolder
     private long count;
     @Getter
     @Setter
-    @Column(columnName = VALUE, isValue = true, function = Function.Avg)
+    @Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE, function = Function.Avg)
     private long value;
 
     @Entrance

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/MaxDoubleMetrics.java

@@ -32,7 +32,7 @@ public abstract class MaxDoubleMetrics extends Metrics implements DoubleValueHol
 
     @Getter
     @Setter
-    @Column(columnName = VALUE, isValue = true)
+    @Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE)
     private double value;
 
     @Entrance

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/MaxLongMetrics.java

@@ -35,7 +35,7 @@ public abstract class MaxLongMetrics extends Metrics implements LongValueHolder
 
     @Getter
     @Setter
-    @Column(columnName = VALUE, isValue = true)
+    @Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE)
     private long value;
 
     @Entrance

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/MinDoubleMetrics.java

@@ -32,7 +32,7 @@ public abstract class MinDoubleMetrics extends Metrics implements DoubleValueHol
 
     @Getter
     @Setter
-    @Column(columnName = VALUE, isValue = true)
+    @Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE)
     private double value = Double.MAX_VALUE;
 
     @Entrance

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/MinLongMetrics.java

@@ -32,7 +32,7 @@ public abstract class MinLongMetrics extends Metrics implements LongValueHolder
 
     @Getter
     @Setter
-    @Column(columnName = VALUE, isValue = true)
+    @Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE)
     private long value = Long.MAX_VALUE;
 
     @Entrance

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentMetrics.java

@@ -38,7 +38,7 @@ public abstract class PercentMetrics extends Metrics implements IntValueHolder {
     private long total;
     @Getter
     @Setter
-    @Column(columnName = PERCENTAGE, isValue = true, function = Function.Avg)
+    @Column(columnName = PERCENTAGE, dataType = Column.ValueDataType.COMMON_VALUE, function = Function.Avg)
     private int percentage;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PercentileMetrics.java

@@ -47,13 +47,9 @@ public abstract class PercentileMetrics extends Metrics implements MultiIntValue
         99
     };
 
-    /**
-     * The special case when the column is isValue = true, but storageOnly = true, because it is {@link DataTable} type,
-     * this column can't be query by the aggregation way.
-     */
     @Getter
     @Setter
-    @Column(columnName = VALUE, isValue = true, storageOnly = true)
+    @Column(columnName = VALUE, dataType = Column.ValueDataType.LABELED_VALUE, storageOnly = true)
     private DataTable percentileValues;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/PxxMetrics.java

@@ -43,7 +43,7 @@ public abstract class PxxMetrics extends Metrics implements IntValueHolder {
 
     @Getter
     @Setter
-    @Column(columnName = VALUE, isValue = true, function = Function.Avg)
+    @Column(columnName = VALUE, dataType = Column.ValueDataType.HISTOGRAM, function = Function.Avg)
     private int value;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/metrics/SumMetrics.java

@@ -33,7 +33,7 @@ public abstract class SumMetrics extends Metrics implements LongValueHolder {
 
     @Getter
     @Setter
-    @Column(columnName = VALUE, isValue = true, function = Function.Sum)
+    @Column(columnName = VALUE, dataType = Column.ValueDataType.COMMON_VALUE, function = Function.Sum)
     private long value;
 
     @Entrance

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/topn/TopN.java

@@ -39,7 +39,7 @@ public abstract class TopN extends Record implements ComparableStorageData {
     private String statement;
     @Getter
     @Setter
-    @Column(columnName = LATENCY)
+    @Column(columnName = LATENCY, dataType = Column.ValueDataType.SAMPLED_RECORD)
     private long latency;
     @Getter
     @Setter

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetricsMetadataQueryService.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.query;
+
+import java.util.Optional;
+import org.apache.skywalking.oap.server.core.query.enumeration.MetricsType;
+import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
+import org.apache.skywalking.oap.server.library.module.Service;
+
+/**
+ * MetricsMetadataQueryService provides the metadata of metrics to other modules.
+ */
+public class MetricsMetadataQueryService implements Service {
+    public MetricsType typeOfMetrics(String metricsName) {
+        final Optional<ValueColumnMetadata.ValueColumn> valueColumn
+            = ValueColumnMetadata.INSTANCE.readValueColumnDefinition(metricsName);
+        if (valueColumn.isPresent()) {
+            switch (valueColumn.get().getDataType()) {
+                case COMMON_VALUE:
+                    return MetricsType.REGULAR_VALUE;
+                case LABELED_VALUE:
+                    return MetricsType.LABELED_VALUE;
+                case HISTOGRAM:
+                    return MetricsType.HEATMAP;
+                case SAMPLED_RECORD:
+                    return MetricsType.SAMPLED_RECORD;
+                case NOT_VALUE:
+                default:
+                    return MetricsType.UNKNOWN;
+            }
+        } else {
+            return MetricsType.UNKNOWN;
+        }
+    }
+}

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/TopNRecordsQueryService.java

@@ -24,6 +24,7 @@
 import org.apache.skywalking.oap.server.core.query.input.TopNCondition;
 import org.apache.skywalking.oap.server.core.query.type.SelectedRecord;
 import org.apache.skywalking.oap.server.core.storage.StorageModule;
+import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
 import org.apache.skywalking.oap.server.core.storage.query.ITopNRecordsQueryDAO;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 import org.apache.skywalking.oap.server.library.module.Service;
@@ -46,6 +47,7 @@ private ITopNRecordsQueryDAO getTopNRecordsQueryDAO() {
     }
 
     public List<SelectedRecord> readSampledRecords(TopNCondition condition, Duration duration) throws IOException {
-        return getTopNRecordsQueryDAO().readSampledRecords(condition, duration);
+        return getTopNRecordsQueryDAO().readSampledRecords(
+            condition, ValueColumnMetadata.INSTANCE.getValueCName(condition.getName()), duration);
     }
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/Column.java

@@ -22,6 +22,7 @@
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
+import lombok.Getter;
 import org.apache.skywalking.oap.server.core.query.sql.Function;
 import org.apache.skywalking.oap.server.core.storage.model.IModelOverride;
 
@@ -37,18 +38,13 @@
      */
     String columnName();
 
-    /**
-     * The column value is used in metrics value query.
-     */
-    boolean isValue() default false;
-
     /**
      * The function is used in aggregation query.
      */
     Function function() default Function.None;
 
     /**
-     * The default value of this column, when its {@link #isValue()} == true.
+     * The default value of this column, when its {@link #dataType()} != {@link ValueDataType#NOT_VALUE}.
      */
     int defaultValue() default 0;
 
@@ -68,4 +64,50 @@
      * @since 7.1.0
      */
     int length() default 200;
+
+    /**
+     * Column with data type != {@link ValueDataType#NOT_VALUE} represents this is a value column. Indicate it would be
+     * queried by UI/CLI.
+     *
+     * @return the data type of this value column. The value column is the query related value Set {@link
+     * ValueDataType#NOT_VALUE} if this is not the value column, read {@link ValueDataType} for more details.
+     * @since 8.0.0
+     */
+    ValueDataType dataType() default ValueDataType.NOT_VALUE;
+
+    /**
+     * ValueDataType represents the data structure of value column. The persistent way of the value column determine the
+     * available ways to query the data.
+     */
+    enum ValueDataType {
+        /**
+         * NOT_VALUE represents this value wouldn't be queried directly through metrics v2 protocol. It could be never
+         * queried, or just through hard code to do so, uch as the lines of topology and service.
+         */
+        NOT_VALUE(false),
+        /**
+         * COMMON_VALUE represents a single value, usually int or long.
+         */
+        COMMON_VALUE(true),
+        /**
+         * LABELLED_VALUE represents this metrics have multiple values with different labels.
+         */
+        LABELED_VALUE(true),
+        /**
+         * HISTOGRAM represents the values are grouped by the buckets, usually suitable for heatmap query.
+         */
+        HISTOGRAM(true),
+        /**
+         * SAMPLED_RECORD represents the values are detail data, being persistent by following some sampled rules.
+         * Usually do topn query based on value column value ASC or DESC.
+         */
+        SAMPLED_RECORD(true);
+
+        @Getter
+        private boolean isValue = false;
+
+        ValueDataType(final boolean isValue) {
+            this.isValue = isValue;
+        }
+    }
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/annotation/ValueColumnMetadata.java

@@ -20,6 +20,8 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Optional;
+import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import org.apache.skywalking.oap.server.core.query.sql.Function;
 
@@ -35,8 +37,12 @@ public enum ValueColumnMetadata {
     /**
      * Register the new metadata for the given model name.
      */
-    public void putIfAbsent(String modelName, String valueCName, Function function, int defaultValue) {
-        mapping.putIfAbsent(modelName, new ValueColumn(valueCName, function, defaultValue));
+    public void putIfAbsent(String modelName,
+                            String valueCName,
+                            Column.ValueDataType dataType,
+                            Function function,
+                            int defaultValue) {
+        mapping.putIfAbsent(modelName, new ValueColumn(valueCName, dataType, function, defaultValue));
     }
 
     /**
@@ -57,6 +63,10 @@ public int getDefaultValue(String metricsName) {
         return findColumn(metricsName).defaultValue;
     }
 
+    public Optional<ValueColumn> readValueColumnDefinition(String metricsName) {
+        return Optional.ofNullable(mapping.get(metricsName));
+    }
+
     private ValueColumn findColumn(String metricsName) {
         ValueColumn column = mapping.get(metricsName);
         if (column == null) {
@@ -65,9 +75,11 @@ private ValueColumn findColumn(String metricsName) {
         return column;
     }
 
+    @Getter
     @RequiredArgsConstructor
-    class ValueColumn {
+    public class ValueColumn {
         private final String valueCName;
+        private final Column.ValueDataType dataType;
         private final Function function;
         private final int defaultValue;
     }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/model/StorageModels.java

@@ -76,13 +76,13 @@ private void retrieval(Class<?> clazz,
                 modelColumns.add(
                     new ModelColumn(
                         new ColumnName(modelName, column.columnName()), field.getType(), column.matchQuery(), column
-                        .storageOnly(), column.isValue(), column.length()));
+                        .storageOnly(), column.dataType().isValue(), column.length()));
                 if (log.isDebugEnabled()) {
                     log.debug("The field named {} with the {} type", column.columnName(), field.getType());
                 }
-                if (column.isValue()) {
+                if (column.dataType().isValue()) {
                     ValueColumnMetadata.INSTANCE.putIfAbsent(
-                        modelName, column.columnName(), column.function(), column.defaultValue());
+                        modelName, column.columnName(), column.dataType(), column.function(), column.defaultValue());
                 }
 
                 List<QueryUnifiedIndex> indexDefinitions = new ArrayList<>();