Skip to content

Commit

Permalink
fix map/struct literal in fe
Browse files Browse the repository at this point in the history
  • Loading branch information
@amorynan
amorynan committed Dec 5, 2023
1 parent 3595f21 commit 7bee504
Show file tree
Hide file tree
Showing 8 changed files with 365 additions and 15 deletions.
17 changes: 17 additions & 0 deletions fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,8 @@ public abstract class Type {
arraySubTypes.add(FLOAT);
arraySubTypes.add(DOUBLE);
arraySubTypes.add(DECIMALV2);
arraySubTypes.add(TIME);
arraySubTypes.add(TIMEV2);
arraySubTypes.add(DATE);
arraySubTypes.add(DATETIME);
arraySubTypes.add(DATEV2);
Expand All @@ -210,6 +212,10 @@ public abstract class Type {
arraySubTypes.add(DECIMAL32);
arraySubTypes.add(DECIMAL64);
arraySubTypes.add(DECIMAL128);
arraySubTypes.add(NULL);
arraySubTypes.add(ARRAY);
arraySubTypes.add(MAP);
arraySubTypes.add(STRUCT);

mapSubTypes = Lists.newArrayList();
mapSubTypes.add(BOOLEAN);
Expand All @@ -220,6 +226,8 @@ public abstract class Type {
mapSubTypes.add(DECIMAL32); // same DEFAULT_DECIMALV3
mapSubTypes.add(DECIMAL64);
mapSubTypes.add(DECIMAL128);
mapSubTypes.add(TIME);
mapSubTypes.add(TIMEV2);
mapSubTypes.add(DATE);
mapSubTypes.add(DATETIME);
mapSubTypes.add(DATEV2);
Expand All @@ -230,6 +238,9 @@ public abstract class Type {
mapSubTypes.add(VARCHAR);
mapSubTypes.add(STRING);
mapSubTypes.add(NULL);
mapSubTypes.add(ARRAY);
mapSubTypes.add(MAP);
mapSubTypes.add(STRUCT);

structSubTypes = Lists.newArrayList();
structSubTypes.add(BOOLEAN);
Expand All @@ -240,6 +251,8 @@ public abstract class Type {
structSubTypes.add(DECIMAL32); // same DEFAULT_DECIMALV3
structSubTypes.add(DECIMAL64);
structSubTypes.add(DECIMAL128);
structSubTypes.add(TIME);
structSubTypes.add(TIMEV2);
structSubTypes.add(DATE);
structSubTypes.add(DATETIME);
structSubTypes.add(DATEV2);
Expand All @@ -249,6 +262,10 @@ public abstract class Type {
structSubTypes.add(CHAR);
structSubTypes.add(VARCHAR);
structSubTypes.add(STRING);
structSubTypes.add(NULL);
structSubTypes.add(ARRAY);
structSubTypes.add(MAP);
structSubTypes.add(STRUCT);
}

public static final Set<Class> DATE_SUPPORTED_JAVA_TYPE = Sets.newHashSet(LocalDate.class, java.util.Date.class,
Expand Down
12 changes: 4 additions & 8 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package org.apache.doris.analysis;

import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.thrift.TExprNode;
Expand Down Expand Up @@ -52,6 +51,9 @@ public ArrayLiteral(LiteralExpr... exprs) throws AnalysisException {
Type itemType = Type.NULL;
boolean containsNull = true;
for (LiteralExpr expr : exprs) {
if (!ArrayType.ARRAY.supportSubType(expr.getType())) {
throw new AnalysisException("Invalid item type in Array, not support " + expr.getType());
}
if (itemType == Type.NULL) {
itemType = expr.getType();
} else {
Expand Down Expand Up @@ -135,13 +137,7 @@ public String getStringValueInFe() {
List<String> list = new ArrayList<>(children.size());
children.forEach(v -> {
// we should use type to decide we output array is suitable for json format
if (!(v instanceof NullLiteral) && v.getType().isScalarType()
&& (Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType()))
|| v.getType() == Type.BOOLEAN)) {
list.add(v.getStringValueInFe());
} else {
list.add(v.getStringValueForArray());
}
list.add(getStringLiteralForComplexType(v));
});
return "[" + StringUtils.join(list, ", ") + "]";
}
Expand Down
15 changes: 15 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,21 @@ public static LiteralExpr create(String value, Type type) throws AnalysisExcepti
return literalExpr;
}


public static String getStringLiteralForComplexType(Expr v) {
if (!(v instanceof NullLiteral) && v.getType().isScalarType()
&& (Type.getNumericTypes().contains((ScalarType) v.getActualScalarType(v.getType()))
|| v.getType() == Type.BOOLEAN)) {
return v.getStringValueInFe();
} else if (v.getType().isComplexType()) {
// these type should also call getStringValueInFe which should handle special case for itself
return v.getStringValueInFe();
} else {
return v.getStringValueForArray();
}
}


/**
* Init LiteralExpr's Type information
* only use in rewrite alias function
Expand Down
26 changes: 24 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,12 @@ public MapLiteral(LiteralExpr... exprs) throws AnalysisException {
// 1. limit key type with map-key support
// 2. check type can be assigment for cast
for (int idx = 0; idx < exprs.length && idx + 1 < exprs.length; idx += 2) {
if (!MapType.MAP.supportSubType(exprs[idx].getType())) {
if (exprs[idx].getType().isComplexType() || !MapType.MAP.supportSubType(exprs[idx].getType())) {
throw new AnalysisException("Invalid key type in Map, not support " + exprs[idx].getType());
}
if (!MapType.MAP.supportSubType(exprs[idx + 1].getType())) {
throw new AnalysisException("Invalid value type in Map, not support " + exprs[idx].getType());
}
boolean enableDecimal256 = SessionVariable.getEnableDecimal256();
keyType = Type.getAssignmentCompatibleType(keyType, exprs[idx].getType(), true, enableDecimal256);
valueType = Type.getAssignmentCompatibleType(valueType, exprs[idx + 1].getType(), true, enableDecimal256);
Expand Down Expand Up @@ -159,7 +162,26 @@ public String getStringValue() {

@Override
public String getStringValueForArray() {
return null;
List<String> list = new ArrayList<>(children.size());
for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) {
list.add(children.get(i).getStringValueForArray() + ":" + children.get(i + 1).getStringValueForArray());
}
return "{" + StringUtils.join(list, ", ") + "}";
}

@Override
public String getStringValueInFe() {
List<String> list = new ArrayList<>(children.size());
for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) {
// we should use type to decide we output array is suitable for json format
if (children.get(i).getType().isComplexType()) {
// map key type do not support complex type
throw new UnsupportedOperationException("Unsupport key type for MAP: " + children.get(i).getType());
}
list.add(children.get(i).getStringValueForArray()
+ ":" + getStringLiteralForComplexType(children.get(i + 1)));
}
return "{" + StringUtils.join(list, ", ") + "}";
}

@Override
Expand Down
15 changes: 12 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ public StructLiteral(LiteralExpr... exprs) throws AnalysisException {
type = new StructType();
children = new ArrayList<>();
for (LiteralExpr expr : exprs) {
if (!expr.getType().isNull() && !type.supportSubType(expr.getType())) {
throw new AnalysisException("Invalid element type in STRUCT.");
if (!StructType.STRUCT.supportSubType(expr.getType())) {
throw new AnalysisException("Invalid element type in STRUCT: " + expr.getType());
}
((StructType) type).addField(new StructField(expr.getType()));
children.add(expr);
Expand Down Expand Up @@ -80,7 +80,16 @@ public String getStringValue() {

@Override
public String getStringValueForArray() {
return null;
List<String> list = new ArrayList<>(children.size());
children.forEach(v -> list.add(v.getStringValueForArray()));
return "{" + StringUtils.join(list, ", ") + "}";
}

@Override
public String getStringValueInFe() {
List<String> list = new ArrayList<>(children.size());
children.forEach(v -> list.add(getStringLiteralForComplexType(v)));
return "{" + StringUtils.join(list, ", ") + "}";
}

@Override
Expand Down
26 changes: 24 additions & 2 deletions fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,16 @@ public void testGetStringValueForArray() throws AnalysisException {

ArrayLiteral arrayLiteral9 = new ArrayLiteral();
Assert.assertEquals("[]", arrayLiteral9.getStringValueForArray());

ArrayLiteral arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral);
MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral);
StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral);
ArrayLiteral arrayLiteral10 = new ArrayLiteral(arrayLiteral, arrayLiteral);
Assert.assertEquals("[[\"1\", \"2.15\"], [\"1\", \"2.15\"]]", arrayLiteral10.getStringValueForArray());
ArrayLiteral arrayLiteral11 = new ArrayLiteral(mapLiteral);
Assert.assertEquals("[{\"1\":\"2.15\"}]", arrayLiteral11.getStringValueForArray());
ArrayLiteral arrayLiteral12 = new ArrayLiteral(structLiteral);
Assert.assertEquals("[{\"1\", \"2.15\", \"2022-10-10\"}]", arrayLiteral12.getStringValueForArray());
}


Expand Down Expand Up @@ -89,7 +99,8 @@ public void testGetStringInFe() throws AnalysisException {
Assert.assertEquals("[\"shortstring\", \"shortstring\"]", arrayLiteral3.getStringValueInFe());

ArrayLiteral arrayLiteral4 = new ArrayLiteral(largeIntLiteral, largeIntLiteral);
Assert.assertEquals("[1000000000000000000000, 1000000000000000000000]", arrayLiteral4.getStringValueInFe());
Assert.assertEquals("[1000000000000000000000, 1000000000000000000000]",
arrayLiteral4.getStringValueInFe());

ArrayLiteral arrayLiteral5 = new ArrayLiteral(nullLiteral, nullLiteral);
Assert.assertEquals("[null, null]", arrayLiteral5.getStringValueInFe());
Expand All @@ -98,7 +109,8 @@ public void testGetStringInFe() throws AnalysisException {
Assert.assertEquals("[\"2022-10-10\", \"2022-10-10\"]", arrayLiteral6.getStringValueInFe());

ArrayLiteral arrayLiteral7 = new ArrayLiteral(datetimeLiteral, datetimeLiteral);
Assert.assertEquals("[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]", arrayLiteral7.getStringValueInFe());
Assert.assertEquals("[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]",
arrayLiteral7.getStringValueInFe());

ArrayLiteral arrayLiteral8 = new ArrayLiteral(arrayLiteral7, arrayLiteral7);
Assert.assertEquals("[[\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"], [\"2022-10-10 12:10:10\", \"2022-10-10 12:10:10\"]]",
Expand All @@ -120,5 +132,15 @@ public void testGetStringInFe() throws AnalysisException {
ArrayLiteral arrayLiteral12 = new ArrayLiteral(nullLiteral, intLiteralWithNull);
Assert.assertEquals("[null, 1]", arrayLiteral12.getStringValueInFe());

ArrayLiteral arrayLiteral = new ArrayLiteral(intLiteral1, floatLiteral);
MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral);
StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral);
ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral);
Assert.assertEquals("[[\"1\", \"2.15\"], [\"1\", \"2.15\"]]", arrayLiteral13.getStringValueForArray());
ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral);
Assert.assertEquals("[{\"1\":\"2.15\"}]", arrayLiteral14.getStringValueForArray());
ArrayLiteral arrayLiteral15 = new ArrayLiteral(structLiteral);
Assert.assertEquals("[{\"1\", \"2.15\", \"2022-10-10\"}]", arrayLiteral15.getStringValueForArray());

}
}
Loading

0 comments on commit 7bee504

Please sign in to comment.