add simdjson lib check micro and fix fe fastjson param

amorynan · Aug 26, 2024 · c617a21 · c617a21
1 parent bd5844e
commit c617a21
Show file tree

Hide file tree

Showing 4 changed files with 49 additions and 1 deletion.
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
@@ -69,6 +69,9 @@ endif()
 # enable glog custom prefix
 add_definitions(-DGLOG_CUSTOM_PREFIX_SUPPORT)
 
+# check eof for simdjson
+add_definitions(-DSIMDJSON_CHECK_EOF)
+
 # Options
 option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON)
 option(USE_LIBCPP "Use libc++" OFF)
@@ -382,7 +385,7 @@ endif()
 #   -O3: Enable all compiler optimizations
 #   -DNDEBUG: Turn off dchecks/asserts/debug only code.
 set(CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
-set(CXX_FLAGS_ASAN "-O0 -fsanitize=address -fsanitize=undefined -fno-strict-aliasing -fno-sanitize=alignment,signed-integer-overflow,float-cast-overflow -DUNDEFINED_BEHAVIOR_SANITIZER -DADDRESS_SANITIZER")
+set(CXX_FLAGS_ASAN "-O0 -fno-inline -fsanitize=address -fsanitize=undefined -fno-strict-aliasing -fno-sanitize=alignment,signed-integer-overflow,float-cast-overflow -DUNDEFINED_BEHAVIOR_SANITIZER -DADDRESS_SANITIZER")
 set(CXX_FLAGS_LSAN "-O0 -fsanitize=leak -DLEAK_SANITIZER")
 ## Use for BE-UT
 set(CXX_FLAGS_ASAN_UT "-O0 -fsanitize=address -DADDRESS_SANITIZER")

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/JsonLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/JsonLiteral.java
@@ -21,7 +21,9 @@
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.types.JsonType;
 
+import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
@@ -32,6 +34,13 @@ public class JsonLiteral extends Literal {
 
     private static final ObjectMapper MAPPER = new ObjectMapper();
 
+    static {
+        // 启用严格的 JSON 验证配置
+        MAPPER.enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS);
+        MAPPER.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION);
+        MAPPER.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
+    }
+
     private final String value;
 
     /**

diff --git a/regression-test/suites/json_p0/test_json_load_and_function.groovy b/regression-test/suites/json_p0/test_json_load_and_function.groovy
@@ -132,6 +132,24 @@ suite("test_json_load_and_function", "p0") {
        success = false
     }
     assertEquals(false, success)
+
+    // deal with tail charactor is not valid json
+    try {
+        sql """INSERT INTO ${testTable} VALUES(26, '{"a": true} "x"')"""
+    } catch(Exception ex) {
+       logger.info("""INSERT INTO ${testTable} invalid json failed: """ + ex)
+    }
+    try {
+        sql """INSERT INTO ${testTable} VALUES(26, '[1]x')"""
+    } catch(Exception ex) {
+       logger.info("""INSERT INTO ${testTable} invalid json failed: """ + ex)
+    }
+    try {
+        sql """INSERT INTO ${testTable} VALUES(26, '{"a":"b"}#')"""
+    } catch(Exception ex) {
+       logger.info("""INSERT INTO ${testTable} invalid json failed: """ + ex)
+    }
+
     success = true
     try {
         sql """INSERT INTO ${testTable} VALUES(26, 'abc')"""

diff --git a/regression-test/suites/nereids_p0/jsonb_p0/test_jsonb_load_and_function.groovy b/regression-test/suites/nereids_p0/jsonb_p0/test_jsonb_load_and_function.groovy
@@ -130,6 +130,24 @@ suite("test_jsonb_load_and_function", "p0") {
        success = false
     }
     assertEquals(false, success)
+
+    // deal with tail charactor is not valid json
+    try {
+        sql """INSERT INTO ${testTable} VALUES(26, '{"a": true} "x"')"""
+    } catch(Exception ex) {
+       logger.info("""INSERT INTO ${testTable} invalid json failed: """ + ex)
+    }
+    try {
+        sql """INSERT INTO ${testTable} VALUES(26, '[1]x')"""
+    } catch(Exception ex) {
+       logger.info("""INSERT INTO ${testTable} invalid json failed: """ + ex)
+    }
+    try {
+        sql """INSERT INTO ${testTable} VALUES(26, '{"a":"b"}#')"""
+    } catch(Exception ex) {
+       logger.info("""INSERT INTO ${testTable} invalid json failed: """ + ex)
+    }
+
     success = true
     try {
         sql """INSERT INTO ${testTable} VALUES(26, 'abc')"""