diff --git a/be/src/vec/exec/format/column_type_convert.cpp b/be/src/vec/exec/format/column_type_convert.cpp index 0442158b690c39..6a705c4bddcee2 100644 --- a/be/src/vec/exec/format/column_type_convert.cpp +++ b/be/src/vec/exec/format/column_type_convert.cpp @@ -19,8 +19,17 @@ namespace doris::vectorized::converter { +const std::set SafeCastString::FALSE_VALUES = {"false", "off", "no", "0", + ""}; + +#define FOR_LOGICAL_INTEGER_TYPES(M) \ + M(TYPE_TINYINT) \ + M(TYPE_SMALLINT) \ + M(TYPE_INT) \ + M(TYPE_BIGINT) \ + M(TYPE_LARGEINT) + #define FOR_LOGICAL_NUMERIC_TYPES(M) \ - M(TYPE_BOOLEAN) \ M(TYPE_TINYINT) \ M(TYPE_SMALLINT) \ M(TYPE_INT) \ @@ -30,7 +39,6 @@ namespace doris::vectorized::converter { M(TYPE_DOUBLE) #define FOR_LOGICAL_DECIMAL_TYPES(M) \ - M(TYPE_DECIMALV2) \ M(TYPE_DECIMAL32) \ M(TYPE_DECIMAL64) \ M(TYPE_DECIMAL128I) \ @@ -126,46 +134,70 @@ static std::unique_ptr _numeric_converter(const TypeDescrip PrimitiveType src_primitive_type = src_type.type; PrimitiveType dst_primitive_type = remove_nullable(dst_type)->get_type_as_type_descriptor().type; - switch (src_primitive_type) { -#define DISPATCH(SRC_PTYPE) \ - case SRC_PTYPE: { \ - switch (dst_primitive_type) { \ - case TYPE_BOOLEAN: \ - return std::make_unique>(); \ + + switch (dst_primitive_type) { +#define DISPATCH(DST_PTYPE) \ + case DST_PTYPE: { \ + switch (src_primitive_type) { \ case TYPE_TINYINT: \ - return std::make_unique>(); \ + return std::make_unique>(); \ case TYPE_SMALLINT: \ - return std::make_unique>(); \ + return std::make_unique>(); \ case TYPE_INT: \ - return std::make_unique>(); \ + return std::make_unique>(); \ case TYPE_BIGINT: \ - return std::make_unique>(); \ + return std::make_unique>(); \ case TYPE_LARGEINT: \ - return std::make_unique>(); \ - case TYPE_FLOAT: \ - return std::make_unique>(); \ - case TYPE_DOUBLE: \ - return std::make_unique>(); \ + return std::make_unique>(); \ default: \ return std::make_unique(src_type, dst_type); \ } \ } - FOR_LOGICAL_NUMERIC_TYPES(DISPATCH) + FOR_LOGICAL_INTEGER_TYPES(DISPATCH) +#undef DISPATCH + + case TYPE_FLOAT: { + switch (src_primitive_type) { +#define DISPATCH(SRC_PTYPE) \ + case SRC_PTYPE: { \ + return std::make_unique>(); \ + } + FOR_LOGICAL_INTEGER_TYPES(DISPATCH) #undef DISPATCH + default: + return std::make_unique(src_type, dst_type); + } + } + + case TYPE_DOUBLE: { + switch (src_primitive_type) { +#define DISPATCH(SRC_PTYPE) \ + case SRC_PTYPE: { \ + return std::make_unique>(); \ + } + FOR_LOGICAL_NUMERIC_TYPES(DISPATCH) +#undef DISPATCH + default: + return std::make_unique(src_type, dst_type); + } + } default: return std::make_unique(src_type, dst_type); } } +template static std::unique_ptr _to_string_converter(const TypeDescriptor& src_type, const DataTypePtr& dst_type) { PrimitiveType src_primitive_type = src_type.type; // numeric type to string, using native std::to_string - if (_is_numeric_type(src_primitive_type)) { + if (src_primitive_type == TYPE_BOOLEAN) { + return std::make_unique(); + } else if (_is_numeric_type(src_primitive_type)) { switch (src_primitive_type) { #define DISPATCH(SRC_PTYPE) \ case SRC_PTYPE: \ - return std::make_unique>(); + return std::make_unique>(); FOR_LOGICAL_NUMERIC_TYPES(DISPATCH) #undef DISPATCH default: @@ -195,14 +227,16 @@ static std::unique_ptr _to_string_converter(const TypeDescr return std::make_unique(src_type, dst_type); } +template static std::unique_ptr _from_string_converter(const TypeDescriptor& src_type, const DataTypePtr& dst_type) { PrimitiveType dst_primitive_type = remove_nullable(dst_type)->get_type_as_type_descriptor().type; switch (dst_primitive_type) { -#define DISPATCH(DST_PTYPE) \ - case DST_PTYPE: \ - return std::make_unique>(remove_nullable(dst_type)); +#define DISPATCH(DST_PTYPE) \ + case DST_PTYPE: \ + return std::make_unique>( \ + remove_nullable(dst_type)); FOR_ALL_LOGICAL_TYPES(DISPATCH) #undef DISPATCH default: @@ -216,24 +250,26 @@ static std::unique_ptr _numeric_to_decimal_converter( PrimitiveType dst_primitive_type = remove_nullable(dst_type)->get_type_as_type_descriptor().type; int scale = remove_nullable(dst_type)->get_scale(); + int precision = remove_nullable(dst_type)->get_precision(); switch (src_primitive_type) { -#define DISPATCH(SRC_PTYPE) \ - case SRC_PTYPE: { \ - switch (dst_primitive_type) { \ - case TYPE_DECIMALV2: \ - return std::make_unique>(scale); \ - case TYPE_DECIMAL32: \ - return std::make_unique>(scale); \ - case TYPE_DECIMAL64: \ - return std::make_unique>(scale); \ - case TYPE_DECIMAL128I: \ - return std::make_unique>( \ - scale); \ - case TYPE_DECIMAL256: \ - return std::make_unique>(scale); \ - default: \ - return std::make_unique(src_type, dst_type); \ - } \ +#define DISPATCH(SRC_PTYPE) \ + case SRC_PTYPE: { \ + switch (dst_primitive_type) { \ + case TYPE_DECIMAL32: \ + return std::make_unique>( \ + precision, scale); \ + case TYPE_DECIMAL64: \ + return std::make_unique>( \ + precision, scale); \ + case TYPE_DECIMAL128I: \ + return std::make_unique>( \ + precision, scale); \ + case TYPE_DECIMAL256: \ + return std::make_unique>( \ + precision, scale); \ + default: \ + return std::make_unique(src_type, dst_type); \ + } \ } FOR_LOGICAL_NUMERIC_TYPES(DISPATCH) #undef DISPATCH @@ -252,8 +288,6 @@ static std::unique_ptr _decimal_to_numeric_converter( #define DISPATCH(DST_PTYPE) \ case DST_PTYPE: { \ switch (src_primitive_type) { \ - case TYPE_DECIMALV2: \ - return std::make_unique>(scale); \ case TYPE_DECIMAL32: \ return std::make_unique>(scale); \ case TYPE_DECIMAL64: \ @@ -274,18 +308,61 @@ static std::unique_ptr _decimal_to_numeric_converter( } } -std::unique_ptr ColumnTypeConverter::get_converter( - const TypeDescriptor& src_type, const DataTypePtr& dst_type) { +static std::unique_ptr _decimal_converter(const TypeDescriptor& src_type, + const DataTypePtr& dst_type) { + int from_precision = src_type.precision; + int from_scale = src_type.scale; + int to_precision = remove_nullable(dst_type)->get_precision(); + int to_scale = remove_nullable(dst_type)->get_scale(); + + if (from_scale == to_scale && from_precision == to_precision) { + return std::make_unique(); + } + PrimitiveType src_primitive_type = src_type.type; PrimitiveType dst_primitive_type = remove_nullable(dst_type)->get_type_as_type_descriptor().type; - if (src_primitive_type == dst_primitive_type) { - return std::make_unique(); + switch (dst_primitive_type) { +#define DISPATCH(DST_PTYPE) \ + case DST_PTYPE: { \ + switch (src_primitive_type) { \ + case TYPE_DECIMAL32: \ + return std::make_unique>( \ + from_precision, from_scale, to_precision, to_scale); \ + case TYPE_DECIMAL64: \ + return std::make_unique>( \ + from_precision, from_scale, to_precision, to_scale); \ + case TYPE_DECIMAL128I: \ + return std::make_unique>( \ + from_precision, from_scale, to_precision, to_scale); \ + case TYPE_DECIMAL256: \ + return std::make_unique>( \ + from_precision, from_scale, to_precision, to_scale); \ + default: \ + return std::make_unique(src_type, dst_type); \ + } \ + } + FOR_LOGICAL_DECIMAL_TYPES(DISPATCH) +#undef DISPATCH + default: + return std::make_unique(src_type, dst_type); } +} + +std::unique_ptr ColumnTypeConverter::get_converter( + const TypeDescriptor& src_type, const DataTypePtr& dst_type, FileFormat file_format) { + PrimitiveType src_primitive_type = src_type.type; + PrimitiveType dst_primitive_type = + remove_nullable(dst_type)->get_type_as_type_descriptor().type; if (is_string_type(src_primitive_type) && is_string_type(dst_primitive_type)) { return std::make_unique(); } + if (_is_decimal_type(src_primitive_type) && _is_decimal_type(dst_primitive_type)) { + return _decimal_converter(src_type, dst_type); + } + + if (src_primitive_type == dst_primitive_type) { return std::make_unique(); } @@ -298,13 +375,21 @@ std::unique_ptr ColumnTypeConverter::get_converter( // change to string type // example: decimal -> string if (is_string_type(dst_primitive_type)) { - return _to_string_converter(src_type, dst_type); + if (file_format == ORC) { + return _to_string_converter(src_type, dst_type); + } else { + return _to_string_converter(src_type, dst_type); + } } // string type to other type // example: string -> date if (is_string_type(src_primitive_type)) { - return _from_string_converter(src_type, dst_type); + if (file_format == ORC) { + return _from_string_converter(src_type, dst_type); + } else { + return _from_string_converter(src_type, dst_type); + } } // date to datetime, datetime to date diff --git a/be/src/vec/exec/format/column_type_convert.h b/be/src/vec/exec/format/column_type_convert.h index d4a8186549ab1d..3052d32aa1a84b 100644 --- a/be/src/vec/exec/format/column_type_convert.h +++ b/be/src/vec/exec/format/column_type_convert.h @@ -26,12 +26,30 @@ namespace doris::vectorized::converter { +enum FileFormat { COMMON, ORC, PARQUET }; + template -constexpr bool is_decimal_type_const() { +constexpr bool is_decimal_type() { return type == TYPE_DECIMALV2 || type == TYPE_DECIMAL32 || type == TYPE_DECIMAL64 || type == TYPE_DECIMAL128I || type == TYPE_DECIMAL256; } +template +constexpr bool is_integer_type() { + return type == TYPE_INT || type == TYPE_TINYINT || type == TYPE_SMALLINT || + type == TYPE_BIGINT || type == TYPE_LARGEINT; +} + +template +constexpr bool is_real_type() { + return type == TYPE_FLOAT || type == TYPE_DOUBLE; +} + +template +constexpr bool is_numeric_type() { + return is_integer_type() || is_real_type(); +} + /** * Unified schema change interface for all format readers: * @@ -55,7 +73,8 @@ class ColumnTypeConverter { * @param dst_type column type from FE planner(the changed column type) */ static std::unique_ptr get_converter(const TypeDescriptor& src_type, - const DataTypePtr& dst_type); + const DataTypePtr& dst_type, + FileFormat file_format); ColumnTypeConverter() = default; virtual ~ColumnTypeConverter() = default; @@ -123,11 +142,14 @@ class UnsupportedConverter : public ColumnTypeConverter { }; template -class NumericToNumericConverter : public ColumnTypeConverter { + requires(is_integer_type() && is_integer_type()) +class IntegerToIntegerConverter : public ColumnTypeConverter { +public: Status convert(ColumnPtr& src_col, MutableColumnPtr& dst_col) override { using SrcColumnType = typename PrimitiveTypeTraits::ColumnType; - using DstCppType = typename PrimitiveTypeTraits::CppType; + using SrcCppType = typename PrimitiveTypeTraits::CppType; using DstColumnType = typename PrimitiveTypeTraits::ColumnType; + using DstCppType = typename PrimitiveTypeTraits::CppType; ColumnPtr from_col = remove_nullable(src_col); MutableColumnPtr to_col = remove_nullable(dst_col->get_ptr())->assume_mutable(); @@ -137,29 +159,140 @@ class NumericToNumericConverter : public ColumnTypeConverter { to_col->resize(start_idx + rows); auto& data = static_cast(*to_col.get()).get_data(); for (int i = 0; i < rows; ++i) { + if constexpr (sizeof(DstCppType) < sizeof(SrcCppType)) { + SrcCppType src_value = src_data[i]; + if ((SrcCppType)std::numeric_limits::min() > src_value || + src_value > (SrcCppType)std::numeric_limits::max()) { + return Status::InternalError("Failed to cast value '{}' to {} column", + src_value, dst_col->get_name()); + } + } + data[start_idx + i] = static_cast(src_data[i]); } + return Status::OK(); + } +}; + +template + requires(is_numeric_type() && is_real_type()) +class NumericToFloatPointConverter : public ColumnTypeConverter { + static constexpr long MIN_EXACT_DOUBLE = -(1L << 52); // -2^52 + static constexpr long MAX_EXACT_DOUBLE = (1L << 52) - 1; // 2^52 - 1 + static constexpr long MIN_EXACT_FLOAT = -(1L << 23); // -2^23 + static constexpr long MAX_EXACT_FLOAT = (1L << 23) - 1; // 2^23 - 1 + + bool overflow(typename PrimitiveTypeTraits::CppType value) const { + if constexpr (DstPrimitiveType == TYPE_DOUBLE) { + return value < MIN_EXACT_DOUBLE || value > MAX_EXACT_DOUBLE; + } else if constexpr (DstPrimitiveType == TYPE_FLOAT) { + return value < MIN_EXACT_FLOAT || value > MAX_EXACT_FLOAT; + } + return true; // Default case, should not occur + } + +public: + Status convert(ColumnPtr& src_col, MutableColumnPtr& dst_col) override { + using SrcColumnType = typename PrimitiveTypeTraits::ColumnType; + using SrcCppType = typename PrimitiveTypeTraits::CppType; + using DstColumnType = typename PrimitiveTypeTraits::ColumnType; + using DstCppType = typename PrimitiveTypeTraits::CppType; + ColumnPtr from_col = remove_nullable(src_col); + MutableColumnPtr to_col = remove_nullable(dst_col->get_ptr())->assume_mutable(); + + NullMap* null_map = nullptr; + if (dst_col->is_nullable()) { + null_map = + &static_cast(dst_col.get())->get_null_map_data(); + } + + size_t rows = from_col->size(); + auto& src_data = static_cast(from_col.get())->get_data(); + size_t start_idx = to_col->size(); + to_col->resize(start_idx + rows); + auto& data = static_cast(*to_col.get()).get_data(); + for (int i = 0; i < rows; ++i) { + SrcCppType src_value = src_data[i]; + if constexpr (is_integer_type()) { + if (overflow(src_value)) { + if (null_map == nullptr) { + return Status::InternalError("Failed to cast value '{}' to {} column", + src_value, dst_col->get_name()); + } else { + (*null_map)[start_idx + i] = 1; + } + } + } + + data[start_idx + i] = static_cast(src_value); + } + return Status::OK(); + } +}; +class BooleanToStringConverter : public ColumnTypeConverter { +public: + Status convert(ColumnPtr& src_col, MutableColumnPtr& dst_col) override { + using SrcColumnType = typename PrimitiveTypeTraits::ColumnType; + ColumnPtr from_col = remove_nullable(src_col); + MutableColumnPtr to_col = remove_nullable(dst_col->get_ptr())->assume_mutable(); + + size_t rows = from_col->size(); + auto& src_data = static_cast(from_col.get())->get_data(); + auto& string_col = static_cast(*to_col.get()); + for (int i = 0; i < rows; ++i) { + std::string value = src_data[i] != 0 ? "TRUE" : "FALSE"; + string_col.insert_data(value.data(), value.size()); + } return Status::OK(); } }; -template +template + requires(is_numeric_type()) class NumericToStringConverter : public ColumnTypeConverter { +private: +public: Status convert(ColumnPtr& src_col, MutableColumnPtr& dst_col) override { using SrcColumnType = typename PrimitiveTypeTraits::ColumnType; ColumnPtr from_col = remove_nullable(src_col); MutableColumnPtr to_col = remove_nullable(dst_col->get_ptr())->assume_mutable(); + NullMap* null_map = nullptr; + if (dst_col->is_nullable()) { + null_map = &reinterpret_cast(dst_col.get()) + ->get_null_map_data(); + } + size_t rows = from_col->size(); + size_t start_idx = to_col->size(); auto& src_data = static_cast(from_col.get())->get_data(); auto& string_col = static_cast(*to_col.get()); for (int i = 0; i < rows; ++i) { - if constexpr (SrcPrimitiveType == TYPE_LARGEINT) { - string value = int128_to_string(src_data[i]); - string_col.insert_data(value.data(), value.size()); + if constexpr (SrcPrimitiveType == TYPE_FLOAT || SrcPrimitiveType == TYPE_DOUBLE) { + if (fileFormat == FileFormat::ORC && std::isnan(src_data[i])) { + if (null_map == nullptr) { + return Status::InternalError("Failed to cast value '{}' to {} column", + src_data[i], dst_col->get_name()); + } else { + (*null_map)[start_idx + i] = 1; + } + } + char buf[128]; + int strlen; + if constexpr (SrcPrimitiveType == TYPE_FLOAT) { + strlen = FastFloatToBuffer(src_data[i], buf); + } else { + strlen = FastDoubleToBuffer(src_data[i], buf); + } + string_col.insert_data(buf, strlen); } else { - string value = std::to_string(src_data[i]); + std::string value; + if constexpr (SrcPrimitiveType == TYPE_LARGEINT) { + value = int128_to_string(src_data[i]); + } else { + value = std::to_string(src_data[i]); + } string_col.insert_data(value.data(), value.size()); } } @@ -193,8 +326,9 @@ class DecimalToStringConverter : public ColumnTypeConverter { } }; -template +template class TimeToStringConverter : public ColumnTypeConverter { +public: Status convert(ColumnPtr& src_col, MutableColumnPtr& dst_col) override { using SrcCppType = typename PrimitiveTypeTraits::CppType; using SrcColumnType = typename PrimitiveTypeTraits::ColumnType; @@ -214,16 +348,33 @@ class TimeToStringConverter : public ColumnTypeConverter { } }; -template +template struct SafeCastString {}; template <> struct SafeCastString { + // Ref: https://github.com/apache/hive/blob/4df4d75bf1e16fe0af75aad0b4179c34c07fc975/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java#L559 + static const std::set FALSE_VALUES; static bool safe_cast_string(const char* startptr, const int buffer_size, PrimitiveTypeTraits::ColumnType::value_type* value) { - int32 cast_to_int = 0; - bool can_cast = safe_strto32(startptr, buffer_size, &cast_to_int); - *value = cast_to_int == 0 ? 0 : 1; + std::string str_value(startptr, buffer_size); + std::transform(str_value.begin(), str_value.end(), str_value.begin(), ::tolower); + bool is_false = (FALSE_VALUES.find(str_value) != FALSE_VALUES.end()); + *value = is_false ? 0 : 1; + return true; + } +}; + +//Apache Hive reads 0 as false, numeric string as true and non-numeric string as null for ORC file format +// https://github.com/apache/orc/blob/fb1c4cb9461d207db652fc253396e57640ed805b/java/core/src/java/org/apache/orc/impl/ConvertTreeReaderFactory.java#L567 +template <> +struct SafeCastString { + static bool safe_cast_string(const char* startptr, const int buffer_size, + PrimitiveTypeTraits::ColumnType::value_type* value) { + std::string str_value(startptr, buffer_size); + int64 cast_to_long = 0; + bool can_cast = safe_strto64(startptr, buffer_size, &cast_to_long); + *value = cast_to_long == 0 ? 0 : 1; return can_cast; } }; @@ -285,23 +436,34 @@ struct SafeCastString { } }; -template <> -struct SafeCastString { +template +struct SafeCastString { static bool safe_cast_string(const char* startptr, const int buffer_size, PrimitiveTypeTraits::ColumnType::value_type* value) { float cast_to_float = 0; bool can_cast = safe_strtof(std::string(startptr, buffer_size), &cast_to_float); + if (can_cast && fileFormat == ORC) { + // Apache Hive reads Float.NaN as null when coerced to varchar for ORC file format. + if (std::isnan(cast_to_float)) { + return false; + } + } *value = cast_to_float; return can_cast; } }; -template <> -struct SafeCastString { +template +struct SafeCastString { static bool safe_cast_string(const char* startptr, const int buffer_size, PrimitiveTypeTraits::ColumnType::value_type* value) { double cast_to_double = 0; bool can_cast = safe_strtod(std::string(startptr, buffer_size), &cast_to_double); + if (can_cast && fileFormat == ORC) { + if (std::isnan(cast_to_double)) { + return false; + } + } *value = cast_to_double; return can_cast; } @@ -357,7 +519,7 @@ struct SafeCastDecimalString { } }; -template +template class CastStringConverter : public ColumnTypeConverter { private: DataTypePtr _dst_type_desc; @@ -387,17 +549,21 @@ class CastStringConverter : public ColumnTypeConverter { DstCppType& value = data[start_idx + i]; auto string_value = string_col.get_data_at(i); bool can_cast = false; - if constexpr (is_decimal_type_const()) { + if constexpr (is_decimal_type()) { can_cast = SafeCastDecimalString::safe_cast_string( string_value.data, string_value.size, &value, _dst_type_desc->get_precision(), _dst_type_desc->get_scale()); } else if constexpr (DstPrimitiveType == TYPE_DATETIMEV2) { can_cast = SafeCastString::safe_cast_string( string_value.data, string_value.size, &value, _dst_type_desc->get_scale()); + } else if constexpr (DstPrimitiveType == TYPE_BOOLEAN && fileFormat == ORC) { + can_cast = SafeCastString::safe_cast_string( + string_value.data, string_value.size, &value); } else { can_cast = SafeCastString::safe_cast_string( string_value.data, string_value.size, &value); } + if (!can_cast) { if (null_map == nullptr) { return Status::InternalError("Failed to cast string '{}' to not null column", @@ -415,6 +581,7 @@ class CastStringConverter : public ColumnTypeConverter { // only support date & datetime v2 template class TimeV2Converter : public ColumnTypeConverter { +public: Status convert(ColumnPtr& src_col, MutableColumnPtr& dst_col) override { using SrcColumnType = typename PrimitiveTypeTraits::ColumnType; using DstColumnType = typename PrimitiveTypeTraits::ColumnType; @@ -442,61 +609,95 @@ class TimeV2Converter : public ColumnTypeConverter { }; template + requires(is_numeric_type() && is_decimal_type()) class NumericToDecimalConverter : public ColumnTypeConverter { private: + int _precision; int _scale; public: - NumericToDecimalConverter(int scale) : _scale(scale) {} + NumericToDecimalConverter(int precision, int scale) : _precision(precision), _scale(scale) {} Status convert(ColumnPtr& src_col, MutableColumnPtr& dst_col) override { using SrcColumnType = typename PrimitiveTypeTraits::ColumnType; + using SrcCppType = typename PrimitiveTypeTraits::CppType; using DstColumnType = typename PrimitiveTypeTraits::ColumnType; using DstNativeType = typename PrimitiveTypeTraits::ColumnType::value_type::NativeType; - using DstCppType = typename PrimitiveTypeTraits::ColumnType::value_type; + using DstDorisType = typename PrimitiveTypeTraits::ColumnType::value_type; ColumnPtr from_col = remove_nullable(src_col); MutableColumnPtr to_col = remove_nullable(dst_col->get_ptr())->assume_mutable(); + NullMap* null_map = nullptr; + if (dst_col->is_nullable()) { + null_map = &reinterpret_cast(dst_col.get()) + ->get_null_map_data(); + } + size_t rows = from_col->size(); auto& src_data = static_cast(from_col.get())->get_data(); size_t start_idx = to_col->size(); to_col->resize(start_idx + rows); auto& data = static_cast(*to_col.get()).get_data(); - int64_t scale_factor = 1; - if (_scale > DecimalV2Value::SCALE) { - scale_factor = common::exp10_i64(_scale - DecimalV2Value::SCALE); - } else if (_scale < DecimalV2Value::SCALE) { - scale_factor = common::exp10_i64(DecimalV2Value::SCALE - _scale); - } + + auto max_result = DataTypeDecimal::get_max_digits_number(_precision); + auto multiplier = DataTypeDecimal::get_scale_multiplier(_scale).value; for (int i = 0; i < rows; ++i) { - if constexpr (SrcPrimitiveType == TYPE_FLOAT || SrcPrimitiveType == TYPE_DOUBLE) { - DecimalV2Value decimal_value; - if constexpr (SrcPrimitiveType == TYPE_FLOAT) { - decimal_value.assign_from_float(src_data[i]); - } else { - decimal_value.assign_from_double(src_data[i]); + const SrcCppType& src_value = src_data[i]; + DstDorisType& res = data[start_idx + i]; + + if constexpr (is_integer_type()) { + if constexpr (sizeof(DstNativeType) < sizeof(SrcCppType)) { + if (src_value > std::numeric_limits::max() || + src_value < std::numeric_limits::min()) { + if (null_map == nullptr) { + return Status::InternalError("Failed to cast value '{}' to {} column", + src_data[i], dst_col->get_name()); + } else { + (*null_map)[start_idx + i] = 1; + } + } } - int128_t decimal_int128 = reinterpret_cast(decimal_value); - if (_scale > DecimalV2Value::SCALE) { - decimal_int128 *= scale_factor; - } else if (_scale < DecimalV2Value::SCALE) { - decimal_int128 /= scale_factor; + if (common::mul_overflow(static_cast(src_value), multiplier, + res.value)) { + if (null_map == nullptr) { + return Status::InternalError("Failed to cast value '{}' to {} column", + src_data[i], dst_col->get_name()); + } else { + (*null_map)[start_idx + i] = 1; + } + } else { + if (res.value > max_result.value || res.value < -max_result.value) { + if (null_map == nullptr) { + return Status::InternalError("Failed to cast value '{}' to {} column", + src_data[i], dst_col->get_name()); + } else { + (*null_map)[start_idx + i] = 1; + } + } } - auto& v = reinterpret_cast(data[start_idx + i]); - v = (DstNativeType)decimal_int128; } else { - data[start_idx + i] = DstCppType::from_int_frac(src_data[i], 0, _scale); + res = static_cast(src_value * multiplier); + if (UNLIKELY(!std::isfinite(src_value) || + src_value * multiplier > max_result.value || + src_value * multiplier < -max_result.value)) { + if (null_map == nullptr) { + return Status::InternalError("Failed to cast value '{}' to {} column", + src_data[i], dst_col->get_name()); + } else { + (*null_map)[start_idx + i] = 1; + } + } } } - return Status::OK(); } }; template + requires(is_numeric_type() && is_decimal_type()) class DecimalToNumericConverter : public ColumnTypeConverter { private: int _scale; @@ -506,6 +707,8 @@ class DecimalToNumericConverter : public ColumnTypeConverter { Status convert(ColumnPtr& src_col, MutableColumnPtr& dst_col) override { using SrcColumnType = typename PrimitiveTypeTraits::ColumnType; + using SrcNativeType = + typename PrimitiveTypeTraits::ColumnType::value_type::NativeType; using DstColumnType = typename PrimitiveTypeTraits::ColumnType; using DstCppType = typename PrimitiveTypeTraits::CppType; @@ -518,13 +721,44 @@ class DecimalToNumericConverter : public ColumnTypeConverter { to_col->resize(start_idx + rows); auto& data = static_cast(*to_col.get()).get_data(); - int64_t scale_factor = common::exp10_i64(_scale); + NullMap* null_map = nullptr; + if (dst_col->is_nullable()) { + null_map = &reinterpret_cast(dst_col.get()) + ->get_null_map_data(); + } + + SrcNativeType scale_factor; + if constexpr (sizeof(SrcNativeType) <= sizeof(int)) { + scale_factor = common::exp10_i32(_scale); + } else if constexpr (sizeof(SrcNativeType) <= sizeof(int64)) { + scale_factor = common::exp10_i64(_scale); + } else if constexpr (sizeof(SrcNativeType) <= sizeof(__int128)) { + scale_factor = common::exp10_i128(_scale); + } else if constexpr (sizeof(SrcNativeType) <= sizeof(wide::Int256)) { + scale_factor = common::exp10_i256(_scale); + } + for (int i = 0; i < rows; ++i) { if constexpr (DstPrimitiveType == TYPE_FLOAT || DstPrimitiveType == TYPE_DOUBLE) { data[start_idx + i] = static_cast(src_data[i].value / (double)scale_factor); } else { - data[start_idx + i] = static_cast(src_data[i].value / scale_factor); + SrcNativeType tmp_value = src_data[i].value / scale_factor; + + if constexpr (sizeof(SrcNativeType) > sizeof(DstCppType)) { + if ((SrcNativeType)std::numeric_limits::min() > tmp_value || + tmp_value > (SrcNativeType)std::numeric_limits::max()) { + if (null_map == nullptr) { + return Status::InternalError("Failed to cast value '{}' to {} column", + src_data[i].to_string(_scale), + dst_col->get_name()); + } else { + (*null_map)[start_idx + i] = 1; + } + } + } + + data[start_idx + i] = static_cast(tmp_value); } } @@ -532,4 +766,90 @@ class DecimalToNumericConverter : public ColumnTypeConverter { } }; +template +class DecimalToDecimalConverter : public ColumnTypeConverter { +private: + int _from_precision; + int _from_scale; + int _to_precision; + int _to_scale; + +public: + DecimalToDecimalConverter(int from_precision, int from_scale, int to_precision, int to_scale) + : _from_precision(from_precision), + _from_scale(from_scale), + _to_precision(to_precision), + _to_scale(to_scale) {} + + Status convert(ColumnPtr& src_col, MutableColumnPtr& dst_col) override { + using SrcColumnType = typename PrimitiveTypeTraits::ColumnType; + using DstColumnType = typename PrimitiveTypeTraits::ColumnType; + using SrcNativeType = typename PrimitiveTypeTraits< + SrcDecimalPrimitiveType>::ColumnType::value_type::NativeType; + using DstNativeType = typename PrimitiveTypeTraits< + DstDecimalPrimitiveType>::ColumnType::value_type::NativeType; + using MaxNativeType = std::conditional_t<(sizeof(SrcNativeType) > sizeof(DstNativeType)), + SrcNativeType, DstNativeType>; + + auto max_result = + DataTypeDecimal>::get_max_digits_number(_to_precision); + bool narrow_integral = (_to_precision - _to_scale) < (_from_precision - _from_scale); + + ColumnPtr from_col = remove_nullable(src_col); + MutableColumnPtr to_col = remove_nullable(dst_col->get_ptr())->assume_mutable(); + + size_t rows = from_col->size(); + auto& src_data = static_cast(from_col.get())->get_data(); + size_t start_idx = to_col->size(); + to_col->resize(start_idx + rows); + auto& data = static_cast(*to_col.get()).get_data(); + + for (int i = 0; i < rows; ++i) { + SrcNativeType src_value = src_data[i].value; + DstNativeType& res_value = data[start_idx + i].value; + + if (_to_scale > _from_scale) { + const MaxNativeType multiplier = + DataTypeDecimal>::get_scale_multiplier(_to_scale - + _from_scale); + MaxNativeType res; + if (common::mul_overflow(src_value, multiplier, res)) { + return Status::InternalError("Failed to cast value '{}' to {} column", + src_data[i].to_string(_from_scale), + dst_col->get_name()); + } else { + if (res > max_result.value || res < -max_result.value) { + return Status::InternalError("Failed to cast value '{}' to {} column", + src_data[i].to_string(_from_scale), + dst_col->get_name()); + } else { + res_value = static_cast(res); + } + } + } else if (_to_scale == _from_scale) { + res_value = src_value; + if (narrow_integral && + (res_value > max_result.value || res_value < -max_result.value)) { + return Status::InternalError("Failed to cast value '{}' to {} column", + src_data[i].to_string(_from_scale), + dst_col->get_name()); + } + } else { + MaxNativeType multiplier = + DataTypeDecimal>::get_scale_multiplier(_from_scale - + _to_scale) + .value; + res_value = src_value / multiplier; + + if (src_value % multiplier != 0 || res_value > max_result.value || + res_value < -max_result.value) { + return Status::InternalError("Failed to cast value '{}' to {} column", + src_data[i].to_string(_from_scale), + dst_col->get_name()); + } + } + } + return Status::OK(); + } +}; } // namespace doris::vectorized::converter diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp index a488bef7455071..00771b20dc2cec 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.cpp +++ b/be/src/vec/exec/format/orc/vorc_reader.cpp @@ -474,20 +474,41 @@ std::tuple convert_to_orc_literal(const orc::Type* type, const auto* value = literal_data.data; try { switch (type->getKind()) { - case orc::TypeKind::BOOLEAN: + case orc::TypeKind::BOOLEAN: { + if (primitive_type != TYPE_BOOLEAN) { + return std::make_tuple(false, orc::Literal(false)); + } return std::make_tuple(true, orc::Literal(bool(*((uint8_t*)value)))); + } case orc::TypeKind::BYTE: - return std::make_tuple(true, orc::Literal(int64_t(*((int8_t*)value)))); case orc::TypeKind::SHORT: - return std::make_tuple(true, orc::Literal(int64_t(*((int16_t*)value)))); case orc::TypeKind::INT: - return std::make_tuple(true, orc::Literal(int64_t(*((int32_t*)value)))); - case orc::TypeKind::LONG: - return std::make_tuple(true, orc::Literal(*((int64_t*)value))); - case orc::TypeKind::FLOAT: - return std::make_tuple(true, orc::Literal(double(*((float*)value)))); - case orc::TypeKind::DOUBLE: - return std::make_tuple(true, orc::Literal(*((double*)value))); + case orc::TypeKind::LONG: { + if constexpr (primitive_type == TYPE_TINYINT) { + return std::make_tuple(true, orc::Literal(int64_t(*((int8_t*)value)))); + } else if constexpr (primitive_type == TYPE_SMALLINT) { + return std::make_tuple(true, orc::Literal(int64_t(*((int16_t*)value)))); + } else if constexpr (primitive_type == TYPE_INT) { + return std::make_tuple(true, orc::Literal(int64_t(*((int32_t*)value)))); + } else if constexpr (primitive_type == TYPE_BIGINT) { + return std::make_tuple(true, orc::Literal(int64_t(*((int64_t*)value)))); + } + return std::make_tuple(false, orc::Literal(false)); + } + case orc::TypeKind::FLOAT: { + if constexpr (primitive_type == TYPE_FLOAT) { + return std::make_tuple(true, orc::Literal(double(*((float*)value)))); + } else if constexpr (primitive_type == TYPE_DOUBLE) { + return std::make_tuple(true, orc::Literal(double(*((double*)value)))); + } + return std::make_tuple(false, orc::Literal(false)); + } + case orc::TypeKind::DOUBLE: { + if (primitive_type == TYPE_DOUBLE) { + return std::make_tuple(true, orc::Literal(*((double*)value))); + } + return std::make_tuple(false, orc::Literal(false)); + } case orc::TypeKind::STRING: [[fallthrough]]; case orc::TypeKind::BINARY: @@ -496,7 +517,11 @@ std::tuple convert_to_orc_literal(const orc::Type* type, // case orc::TypeKind::CHAR: // [[fallthrough]]; case orc::TypeKind::VARCHAR: { - return std::make_tuple(true, orc::Literal(literal_data.data, literal_data.size)); + if (primitive_type == TYPE_STRING || primitive_type == TYPE_CHAR || + primitive_type == TYPE_VARCHAR) { + return std::make_tuple(true, orc::Literal(literal_data.data, literal_data.size)); + } + return std::make_tuple(false, orc::Literal(false)); } case orc::TypeKind::DECIMAL: { int128_t decimal_value; @@ -508,8 +533,10 @@ std::tuple convert_to_orc_literal(const orc::Type* type, decimal_value = *((int32_t*)value); } else if constexpr (primitive_type == TYPE_DECIMAL64) { decimal_value = *((int64_t*)value); - } else { + } else if constexpr (primitive_type == TYPE_DECIMAL128I) { decimal_value = *((int128_t*)value); + } else { + return std::make_tuple(false, orc::Literal(false)); } return std::make_tuple(true, orc::Literal(orc::Int128(uint64_t(decimal_value >> 64), uint64_t(decimal_value)), @@ -523,12 +550,14 @@ std::tuple convert_to_orc_literal(const orc::Type* type, cctz::civil_day civil_date(date_v1.year(), date_v1.month(), date_v1.day()); day_offset = cctz::convert(civil_date, utc0).time_since_epoch().count() / (24 * 60 * 60); - } else { // primitive_type == TYPE_DATEV2 + } else if (primitive_type == TYPE_DATEV2) { const DateV2Value date_v2 = *reinterpret_cast*>(value); cctz::civil_day civil_date(date_v2.year(), date_v2.month(), date_v2.day()); day_offset = cctz::convert(civil_date, utc0).time_since_epoch().count() / (24 * 60 * 60); + } else { + return std::make_tuple(false, orc::Literal(false)); } return std::make_tuple(true, orc::Literal(orc::PredicateDataType::DATE, day_offset)); } @@ -545,7 +574,7 @@ std::tuple convert_to_orc_literal(const orc::Type* type, datetime_v1.minute(), datetime_v1.second()); seconds = cctz::convert(civil_seconds, utc0).time_since_epoch().count(); nanos = 0; - } else { // primitive_type == TYPE_DATETIMEV2 + } else if (primitive_type == TYPE_DATETIMEV2) { const DateV2Value datetime_v2 = *reinterpret_cast*>(value); cctz::civil_second civil_seconds(datetime_v2.year(), datetime_v2.month(), @@ -553,6 +582,8 @@ std::tuple convert_to_orc_literal(const orc::Type* type, datetime_v2.minute(), datetime_v2.second()); seconds = cctz::convert(civil_seconds, utc0).time_since_epoch().count(); nanos = datetime_v2.microsecond() * 1000; + } else { + return std::make_tuple(false, orc::Literal(false)); } return std::make_tuple(true, orc::Literal(seconds, nanos)); } @@ -1754,7 +1785,8 @@ Status OrcReader::_orc_column_to_doris_column(const std::string& col_name, Colum if (!_converters.contains(converter_key)) { std::unique_ptr converter = - converter::ColumnTypeConverter::get_converter(src_type, data_type); + converter::ColumnTypeConverter::get_converter(src_type, data_type, + converter::FileFormat::ORC); if (!converter->support()) { return Status::InternalError( "The column type of '{}' has changed and is not supported: ", col_name, diff --git a/be/src/vec/exec/format/parquet/parquet_column_convert.cpp b/be/src/vec/exec/format/parquet/parquet_column_convert.cpp index 49636d809aa0d8..80baa4b7f05d36 100644 --- a/be/src/vec/exec/format/parquet/parquet_column_convert.cpp +++ b/be/src/vec/exec/format/parquet/parquet_column_convert.cpp @@ -25,7 +25,6 @@ namespace doris::vectorized::parquet { const cctz::time_zone ConvertParams::utc0 = cctz::utc_time_zone(); #define FOR_LOGICAL_DECIMAL_TYPES(M) \ - M(TYPE_DECIMALV2) \ M(TYPE_DECIMAL32) \ M(TYPE_DECIMAL64) \ M(TYPE_DECIMAL128I) \ @@ -133,8 +132,8 @@ static void get_decimal_converter(FieldSchema* field_schema, TypeDescriptor src_ std::unique_ptr& physical_converter) { const tparquet::SchemaElement& parquet_schema = field_schema->parquet_schema; if (is_decimal(remove_nullable(dst_logical_type))) { - // using destination decimal type, avoid type and scale change - src_logical_type = remove_nullable(dst_logical_type)->get_type_as_type_descriptor(); + src_logical_type = create_decimal(parquet_schema.precision, parquet_schema.scale, false) + ->get_type_as_type_descriptor(); } tparquet::Type::type src_physical_type = parquet_schema.type; @@ -298,8 +297,8 @@ std::unique_ptr PhysicalToLogicalConverter::get_conv if (physical_converter->support()) { physical_converter->_convert_params = std::move(convert_params); - physical_converter->_logical_converter = - converter::ColumnTypeConverter::get_converter(src_logical_type, dst_logical_type); + physical_converter->_logical_converter = converter::ColumnTypeConverter::get_converter( + src_logical_type, dst_logical_type, converter::FileFormat::PARQUET); if (!physical_converter->_logical_converter->support()) { physical_converter.reset(new UnsupportedConverter( "Unsupported type change: " + diff --git a/be/src/vec/exec/format/parquet/parquet_column_convert.h b/be/src/vec/exec/format/parquet/parquet_column_convert.h index d35a69ff59c625..546aa8b8f6fda7 100644 --- a/be/src/vec/exec/format/parquet/parquet_column_convert.h +++ b/be/src/vec/exec/format/parquet/parquet_column_convert.h @@ -154,7 +154,7 @@ struct ConvertParams { * * Ultimate performance optimization: * 1. If process of (First => Second) is consistent, eg. from BYTE_ARRAY to string, no additional copies and conversions will be introduced; - * 2. If process of (Second => Third) is consistent, eg. from decimal(12, 4) to decimal(8, 2), no additional copies and conversions will be introduced; + * 2. If process of (Second => Third) is consistent, no additional copies and conversions will be introduced; * 3. Null map is share among all processes, no additional copies and conversions will be introduced in null map; * 4. Only create one physical column in physical conversion, and reused in each loop; * 5. Only create one logical column in logical conversion, and reused in each loop; diff --git a/be/test/vec/exec/parquet/parquet_thrift_test.cpp b/be/test/vec/exec/parquet/parquet_thrift_test.cpp index 3f3fa3b0f2eff1..c2c072d844ff07 100644 --- a/be/test/vec/exec/parquet/parquet_thrift_test.cpp +++ b/be/test/vec/exec/parquet/parquet_thrift_test.cpp @@ -292,8 +292,8 @@ static doris::TupleDescriptor* create_tuple_desc( for (int i = 0; i < column_descs.size(); ++i) { TSlotDescriptor t_slot_desc; - if (column_descs[i].type == TYPE_DECIMALV2) { - t_slot_desc.__set_slotType(TypeDescriptor::create_decimalv2_type(27, 9).to_thrift()); + if (column_descs[i].type == TYPE_DECIMAL128I) { + t_slot_desc.__set_slotType(TypeDescriptor::create_decimalv3_type(27, 9).to_thrift()); } else { TypeDescriptor descriptor(column_descs[i].type); if (column_descs[i].precision >= 0 && column_descs[i].scale >= 0) { @@ -356,7 +356,7 @@ static void create_block(std::unique_ptr& block) { {"binary_col", TYPE_STRING, sizeof(StringRef), true}, // 64-bit-length, see doris::get_slot_size in primitive_type.cpp {"timestamp_col", TYPE_DATETIMEV2, sizeof(int128_t), true}, - {"decimal_col", TYPE_DECIMALV2, sizeof(DecimalV2Value), true}, + {"decimal_col", TYPE_DECIMAL128I, sizeof(Decimal128V3), true}, {"char_col", TYPE_CHAR, sizeof(StringRef), true}, {"varchar_col", TYPE_VARCHAR, sizeof(StringRef), true}, {"date_col", TYPE_DATEV2, sizeof(uint32_t), true}, diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run75.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run75.hql new file mode 100644 index 00000000000000..41db62fbaba961 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run75.hql @@ -0,0 +1,515 @@ +create database if not exists schema_change; +use schema_change; + +CREATE TABLE IF NOT EXISTS parquet_primitive_types_to_boolean ( + id INT, + bool_col BOOLEAN, + int_col BOOLEAN, + smallint_col BOOLEAN, + tinyint_col BOOLEAN, + bigint_col BOOLEAN, + float_col BOOLEAN, + double_col BOOLEAN, + string_col BOOLEAN, + char1_col BOOLEAN, + char2_col BOOLEAN, + varchar_col BOOLEAN, + date_col BOOLEAN, + timestamp_col BOOLEAN, + decimal1_col BOOLEAN, + decimal2_col BOOLEAN +) STORED AS PARQUET +LOCATION '/user/doris/preinstalled_data/parquet_table/parquet_schema_change'; + +CREATE TABLE IF NOT EXISTS parquet_primitive_types_to_bigint ( + id INT, + bool_col BIGINT, + int_col BIGINT, + smallint_col BIGINT, + tinyint_col BIGINT, + bigint_col BIGINT, + float_col BIGINT, + double_col BIGINT, + string_col BIGINT, + char1_col BIGINT, + char2_col BIGINT, + varchar_col BIGINT, + date_col BIGINT, + timestamp_col BIGINT, + decimal1_col BIGINT, + decimal2_col BIGINT +) STORED AS PARQUET +LOCATION '/user/doris/preinstalled_data/parquet_table/parquet_schema_change'; + +CREATE TABLE IF NOT EXISTS parquet_primitive_types_to_int ( + id INT, + bool_col INT, + int_col INT, + smallint_col INT, + tinyint_col INT, + bigint_col INT, + float_col INT, + double_col INT, + string_col INT, + char1_col INT, + char2_col INT, + varchar_col INT, + date_col INT, + timestamp_col INT, + decimal1_col INT, + decimal2_col INT +) STORED AS PARQUET +LOCATION '/user/doris/preinstalled_data/parquet_table/parquet_schema_change'; + +CREATE TABLE IF NOT EXISTS parquet_primitive_types_to_smallint ( + id INT, + bool_col SMALLINT, + int_col SMALLINT, + smallint_col SMALLINT, + tinyint_col SMALLINT, + bigint_col SMALLINT, + float_col SMALLINT, + double_col SMALLINT, + string_col SMALLINT, + char1_col SMALLINT, + char2_col SMALLINT, + varchar_col SMALLINT, + date_col SMALLINT, + timestamp_col SMALLINT, + decimal1_col SMALLINT, + decimal2_col SMALLINT +) STORED AS PARQUET +LOCATION '/user/doris/preinstalled_data/parquet_table/parquet_schema_change'; + +CREATE TABLE IF NOT EXISTS parquet_primitive_types_to_tinyint ( + id INT, + bool_col TINYINT, + int_col TINYINT, + smallint_col TINYINT, + tinyint_col TINYINT, + bigint_col TINYINT, + float_col TINYINT, + double_col TINYINT, + string_col TINYINT, + char1_col TINYINT, + char2_col TINYINT, + varchar_col TINYINT, + date_col TINYINT, + timestamp_col TINYINT, + decimal1_col TINYINT, + decimal2_col TINYINT +) STORED AS PARQUET +LOCATION '/user/doris/preinstalled_data/parquet_table/parquet_schema_change'; + +CREATE TABLE IF NOT EXISTS parquet_primitive_types_to_float ( + id INT, + bool_col FLOAT, + int_col FLOAT, + smallint_col FLOAT, + tinyint_col FLOAT, + bigint_col FLOAT, + float_col FLOAT, + double_col FLOAT, + string_col FLOAT, + char1_col FLOAT, + char2_col FLOAT, + varchar_col FLOAT, + date_col FLOAT, + timestamp_col FLOAT, + decimal1_col FLOAT, + decimal2_col FLOAT +) STORED AS PARQUET +LOCATION '/user/doris/preinstalled_data/parquet_table/parquet_schema_change'; + +CREATE TABLE IF NOT EXISTS parquet_primitive_types_to_double ( + id INT, + bool_col DOUBLE, + int_col DOUBLE, + smallint_col DOUBLE, + tinyint_col DOUBLE, + bigint_col DOUBLE, + float_col DOUBLE, + double_col DOUBLE, + string_col DOUBLE, + char1_col DOUBLE, + char2_col DOUBLE, + varchar_col DOUBLE, + date_col DOUBLE, + timestamp_col DOUBLE, + decimal1_col DOUBLE, + decimal2_col DOUBLE +) STORED AS PARQUET +LOCATION '/user/doris/preinstalled_data/parquet_table/parquet_schema_change'; + +CREATE TABLE IF NOT EXISTS parquet_primitive_types_to_string ( + id INT, + bool_col STRING, + int_col STRING, + smallint_col STRING, + tinyint_col STRING, + bigint_col STRING, + float_col STRING, + double_col STRING, + string_col STRING, + char1_col STRING, + char2_col STRING, + varchar_col STRING, + date_col STRING, + timestamp_col STRING, + decimal1_col STRING, + decimal2_col STRING +) STORED AS PARQUET +LOCATION '/user/doris/preinstalled_data/parquet_table/parquet_schema_change'; + +CREATE TABLE IF NOT EXISTS parquet_primitive_types_to_date ( + id INT, + bool_col DATE, + int_col DATE, + smallint_col DATE, + tinyint_col DATE, + bigint_col DATE, + float_col DATE, + double_col DATE, + string_col DATE, + char1_col DATE, + char2_col DATE, + varchar_col DATE, + date_col DATE, + timestamp_col DATE, + decimal1_col DATE, + decimal2_col DATE +) STORED AS PARQUET +LOCATION '/user/doris/preinstalled_data/parquet_table/parquet_schema_change'; + +CREATE TABLE IF NOT EXISTS parquet_primitive_types_to_timestamp ( + id INT, + bool_col TIMESTAMP, + int_col TIMESTAMP, + smallint_col TIMESTAMP, + tinyint_col TIMESTAMP, + bigint_col TIMESTAMP, + float_col TIMESTAMP, + double_col TIMESTAMP, + string_col TIMESTAMP, + char1_col TIMESTAMP, + char2_col TIMESTAMP, + varchar_col TIMESTAMP, + date_col TIMESTAMP, + timestamp_col TIMESTAMP, + decimal1_col TIMESTAMP, + decimal2_col TIMESTAMP +) STORED AS PARQUET +LOCATION '/user/doris/preinstalled_data/parquet_table/parquet_schema_change'; + + +CREATE TABLE IF NOT EXISTS parquet_primitive_types_to_decimal1 ( + id INT, + bool_col DECIMAL(20,5), + int_col DECIMAL(20,5), + smallint_col DECIMAL(20,5), + tinyint_col DECIMAL(20,5), + bigint_col DECIMAL(20,5), + float_col DECIMAL(20,5), + double_col DECIMAL(20,5), + string_col DECIMAL(20,5), + char1_col DECIMAL(20,5), + char2_col DECIMAL(20,5), + varchar_col DECIMAL(20,5), + date_col DECIMAL(20,5), + timestamp_col DECIMAL(20,5), + decimal1_col DECIMAL(20,5), + decimal2_col DECIMAL(20,5) +) STORED AS PARQUET +LOCATION '/user/doris/preinstalled_data/parquet_table/parquet_schema_change'; + +CREATE TABLE IF NOT EXISTS parquet_primitive_types_to_decimal2 ( + id INT, + bool_col DECIMAL(7,1), + int_col DECIMAL(7,1), + smallint_col DECIMAL(7,1), + tinyint_col DECIMAL(7,1), + bigint_col DECIMAL(7,1), + float_col DECIMAL(7,1), + double_col DECIMAL(7,1), + string_col DECIMAL(7,1), + char1_col DECIMAL(7,1), + char2_col DECIMAL(7,1), + varchar_col DECIMAL(7,1), + date_col DECIMAL(7,1), + timestamp_col DECIMAL(7,1), + decimal1_col DECIMAL(7,1), + decimal2_col DECIMAL(7,1) +) STORED AS PARQUET +LOCATION '/user/doris/preinstalled_data/parquet_table/parquet_schema_change'; + + + + +CREATE TABLE IF NOT EXISTS orc_primitive_types_to_boolean ( + id INT, + bool_col BOOLEAN, + int_col BOOLEAN, + smallint_col BOOLEAN, + tinyint_col BOOLEAN, + bigint_col BOOLEAN, + float_col BOOLEAN, + double_col BOOLEAN, + string_col BOOLEAN, + char1_col BOOLEAN, + char2_col BOOLEAN, + varchar_col BOOLEAN, + date_col BOOLEAN, + timestamp_col BOOLEAN, + decimal1_col BOOLEAN, + decimal2_col BOOLEAN +) STORED AS orc +LOCATION '/user/doris/preinstalled_data/orc_table/orc_schema_change'; + + +CREATE TABLE IF NOT EXISTS orc_primitive_types_to_bigint ( + id INT, + bool_col BIGINT, + int_col BIGINT, + smallint_col BIGINT, + tinyint_col BIGINT, + bigint_col BIGINT, + float_col BIGINT, + double_col BIGINT, + string_col BIGINT, + char1_col BIGINT, + char2_col BIGINT, + varchar_col BIGINT, + date_col BIGINT, + timestamp_col BIGINT, + decimal1_col BIGINT, + decimal2_col BIGINT +) STORED AS orc +LOCATION '/user/doris/preinstalled_data/orc_table/orc_schema_change'; + +CREATE TABLE IF NOT EXISTS orc_primitive_types_to_int ( + id INT, + bool_col INT, + int_col INT, + smallint_col INT, + tinyint_col INT, + bigint_col INT, + float_col INT, + double_col INT, + string_col INT, + char1_col INT, + char2_col INT, + varchar_col INT, + date_col INT, + timestamp_col INT, + decimal1_col INT, + decimal2_col INT +) STORED AS orc +LOCATION '/user/doris/preinstalled_data/orc_table/orc_schema_change'; + +CREATE TABLE IF NOT EXISTS orc_primitive_types_to_smallint ( + id INT, + bool_col SMALLINT, + int_col SMALLINT, + smallint_col SMALLINT, + tinyint_col SMALLINT, + bigint_col SMALLINT, + float_col SMALLINT, + double_col SMALLINT, + string_col SMALLINT, + char1_col SMALLINT, + char2_col SMALLINT, + varchar_col SMALLINT, + date_col SMALLINT, + timestamp_col SMALLINT, + decimal1_col SMALLINT, + decimal2_col SMALLINT +) STORED AS orc +LOCATION '/user/doris/preinstalled_data/orc_table/orc_schema_change'; + +CREATE TABLE IF NOT EXISTS orc_primitive_types_to_tinyint ( + id INT, + bool_col TINYINT, + int_col TINYINT, + smallint_col TINYINT, + tinyint_col TINYINT, + bigint_col TINYINT, + float_col TINYINT, + double_col TINYINT, + string_col TINYINT, + char1_col TINYINT, + char2_col TINYINT, + varchar_col TINYINT, + date_col TINYINT, + timestamp_col TINYINT, + decimal1_col TINYINT, + decimal2_col TINYINT +) STORED AS orc +LOCATION '/user/doris/preinstalled_data/orc_table/orc_schema_change'; + +CREATE TABLE IF NOT EXISTS orc_primitive_types_to_float ( + id INT, + bool_col FLOAT, + int_col FLOAT, + smallint_col FLOAT, + tinyint_col FLOAT, + bigint_col FLOAT, + float_col FLOAT, + double_col FLOAT, + string_col FLOAT, + char1_col FLOAT, + char2_col FLOAT, + varchar_col FLOAT, + date_col FLOAT, + timestamp_col FLOAT, + decimal1_col FLOAT, + decimal2_col FLOAT +) STORED AS orc +LOCATION '/user/doris/preinstalled_data/orc_table/orc_schema_change'; + +CREATE TABLE IF NOT EXISTS orc_primitive_types_to_double ( + id INT, + bool_col DOUBLE, + int_col DOUBLE, + smallint_col DOUBLE, + tinyint_col DOUBLE, + bigint_col DOUBLE, + float_col DOUBLE, + double_col DOUBLE, + string_col DOUBLE, + char1_col DOUBLE, + char2_col DOUBLE, + varchar_col DOUBLE, + date_col DOUBLE, + timestamp_col DOUBLE, + decimal1_col DOUBLE, + decimal2_col DOUBLE +) STORED AS orc +LOCATION '/user/doris/preinstalled_data/orc_table/orc_schema_change'; + +CREATE TABLE IF NOT EXISTS orc_primitive_types_to_string ( + id INT, + bool_col STRING, + int_col STRING, + smallint_col STRING, + tinyint_col STRING, + bigint_col STRING, + float_col STRING, + double_col STRING, + string_col STRING, + char1_col STRING, + char2_col STRING, + varchar_col STRING, + date_col STRING, + timestamp_col STRING, + decimal1_col STRING, + decimal2_col STRING +) STORED AS orc +LOCATION '/user/doris/preinstalled_data/orc_table/orc_schema_change'; + +CREATE TABLE IF NOT EXISTS orc_primitive_types_to_date ( + id INT, + bool_col DATE, + int_col DATE, + smallint_col DATE, + tinyint_col DATE, + bigint_col DATE, + float_col DATE, + double_col DATE, + string_col DATE, + char1_col DATE, + char2_col DATE, + varchar_col DATE, + date_col DATE, + timestamp_col DATE, + decimal1_col DATE, + decimal2_col DATE +) STORED AS orc +LOCATION '/user/doris/preinstalled_data/orc_table/orc_schema_change'; + +CREATE TABLE IF NOT EXISTS orc_primitive_types_to_timestamp ( + id INT, + bool_col TIMESTAMP, + int_col TIMESTAMP, + smallint_col TIMESTAMP, + tinyint_col TIMESTAMP, + bigint_col TIMESTAMP, + float_col TIMESTAMP, + double_col TIMESTAMP, + string_col TIMESTAMP, + char1_col TIMESTAMP, + char2_col TIMESTAMP, + varchar_col TIMESTAMP, + date_col TIMESTAMP, + timestamp_col TIMESTAMP, + decimal1_col TIMESTAMP, + decimal2_col TIMESTAMP +) STORED AS orc +LOCATION '/user/doris/preinstalled_data/orc_table/orc_schema_change'; + + +CREATE TABLE IF NOT EXISTS orc_primitive_types_to_decimal1 ( + id INT, + bool_col DECIMAL(20,5), + int_col DECIMAL(20,5), + smallint_col DECIMAL(20,5), + tinyint_col DECIMAL(20,5), + bigint_col DECIMAL(20,5), + float_col DECIMAL(20,5), + double_col DECIMAL(20,5), + string_col DECIMAL(20,5), + char1_col DECIMAL(20,5), + char2_col DECIMAL(20,5), + varchar_col DECIMAL(20,5), + date_col DECIMAL(20,5), + timestamp_col DECIMAL(20,5), + decimal1_col DECIMAL(20,5), + decimal2_col DECIMAL(20,5) +) STORED AS orc +LOCATION '/user/doris/preinstalled_data/orc_table/orc_schema_change'; + +CREATE TABLE IF NOT EXISTS orc_primitive_types_to_decimal2 ( + id INT, + bool_col DECIMAL(7,1), + int_col DECIMAL(7,1), + smallint_col DECIMAL(7,1), + tinyint_col DECIMAL(7,1), + bigint_col DECIMAL(7,1), + float_col DECIMAL(7,1), + double_col DECIMAL(7,1), + string_col DECIMAL(7,1), + char1_col DECIMAL(7,1), + char2_col DECIMAL(7,1), + varchar_col DECIMAL(7,1), + date_col DECIMAL(7,1), + timestamp_col DECIMAL(7,1), + decimal1_col DECIMAL(7,1), + decimal2_col DECIMAL(7,1) +) STORED AS orc +LOCATION '/user/doris/preinstalled_data/orc_table/orc_schema_change'; + + +MSCK REPAIR TABLE parquet_primitive_types_to_boolean; +MSCK REPAIR TABLE parquet_primitive_types_to_bigint; +MSCK REPAIR TABLE parquet_primitive_types_to_int; +MSCK REPAIR TABLE parquet_primitive_types_to_smallint; +MSCK REPAIR TABLE parquet_primitive_types_to_tinyint; +MSCK REPAIR TABLE parquet_primitive_types_to_float; +MSCK REPAIR TABLE parquet_primitive_types_to_double; +MSCK REPAIR TABLE parquet_primitive_types_to_string; +MSCK REPAIR TABLE parquet_primitive_types_to_date; +MSCK REPAIR TABLE parquet_primitive_types_to_timestamp; +MSCK REPAIR TABLE parquet_primitive_types_to_decimal1; +MSCK REPAIR TABLE parquet_primitive_types_to_decimal2; + +MSCK REPAIR TABLE orc_primitive_types_to_boolean; +MSCK REPAIR TABLE orc_primitive_types_to_bigint; +MSCK REPAIR TABLE orc_primitive_types_to_int; +MSCK REPAIR TABLE orc_primitive_types_to_smallint; +MSCK REPAIR TABLE orc_primitive_types_to_tinyint; +MSCK REPAIR TABLE orc_primitive_types_to_float; +MSCK REPAIR TABLE orc_primitive_types_to_double; +MSCK REPAIR TABLE orc_primitive_types_to_string; +MSCK REPAIR TABLE orc_primitive_types_to_date; +MSCK REPAIR TABLE orc_primitive_types_to_timestamp; +MSCK REPAIR TABLE orc_primitive_types_to_decimal1; +MSCK REPAIR TABLE orc_primitive_types_to_decimal2; \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_schema_change/origin_file.orc b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_schema_change/origin_file.orc new file mode 100644 index 00000000000000..ab1af67fcf549c Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_schema_change/origin_file.orc differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_schema_change/origin_file.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_schema_change/origin_file.parquet new file mode 100644 index 00000000000000..99bff52b4648af Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_schema_change/origin_file.parquet differ diff --git a/regression-test/data/external_table_p0/hive/test_hive_schema_change_orc.out b/regression-test/data/external_table_p0/hive/test_hive_schema_change_orc.out new file mode 100644 index 00000000000000..1028bdad18420a --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_hive_schema_change_orc.out @@ -0,0 +1,1611 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !orc_boolean_to_boolean -- +true +false +true +false +false +false +true +\N +true +true +true +true + +-- !orc_string_to_boolean -- +\N +\N +\N +true +true +\N +\N +\N +true +\N +\N +\N + +-- !orc_char1_to_boolean -- +\N +\N +\N +true +true +true +true +\N +true +\N +\N +\N + +-- !orc_char2_to_boolean -- +\N +\N +\N +\N +\N +true +true +\N +true +\N +\N +\N + +-- !orc_varchar_to_boolean -- +\N +\N +\N +\N +\N +true +true +\N +true +\N +\N +\N + +-- !orc_tinyint_to_smallint -- +127 +-128 +0 +1 +-2 +120 +-120 +\N +123 +-56 +21 +-61 + +-- !orc_smallint_to_smallint -- +32767 +-32768 +0 +1 +-2 +12345 +-12345 +\N +123 +456 +789 +707 + +-- !orc_string_to_smallint -- +\N +\N +\N +21 +127 +\N +\N +\N +123 +\N +\N +\N + +-- !orc_char1_to_smallint -- +\N +\N +\N +4578 +\N +1 +2 +\N +-123 +\N +\N +\N + +-- !orc_char2_to_smallint -- +\N +\N +\N +\N +\N +3 +3 +\N +123 +\N +\N +\N + +-- !orc_varchar_to_smallint -- +\N +\N +\N +\N +\N +1 +4 +\N +123 +\N +\N +\N + +-- !orc_decimal1_to_smallint -- +\N +\N +0 +1 +1 +\N +0 +\N +123 +456 +789 +707 + +-- !orc_decimal2_to_smallint -- +\N +\N +0 +1 +2 +\N +2 +\N +\N +456 +789 +707 + +-- !orc_tinyint_to_tinyint -- +127 +-128 +0 +1 +-2 +120 +-120 +\N +123 +-56 +21 +-61 + +-- !orc_string_to_tinyint -- +\N +\N +\N +21 +127 +\N +\N +\N +123 +\N +\N +\N + +-- !orc_char1_to_tinyint -- +\N +\N +\N +\N +\N +1 +2 +\N +-123 +\N +\N +\N + +-- !orc_char2_to_tinyint -- +\N +\N +\N +\N +\N +3 +3 +\N +123 +\N +\N +\N + +-- !orc_varchar_to_tinyint -- +\N +\N +\N +\N +\N +1 +4 +\N +123 +\N +\N +\N + +-- !orc_decimal1_to_tinyint -- +\N +\N +0 +1 +1 +\N +0 +\N +123 +\N +\N +\N + +-- !orc_decimal2_to_tinyint -- +\N +\N +0 +1 +2 +\N +2 +\N +\N +\N +\N +\N + +-- !orc_tinyint_to_smallint -- +127 +-128 +0 +1 +-2 +120 +-120 +\N +123 +-56 +21 +-61 + +-- !orc_smallint_to_smallint -- +32767 +-32768 +0 +1 +-2 +12345 +-12345 +\N +123 +456 +789 +707 + +-- !orc_string_to_smallint -- +\N +\N +\N +21 +127 +\N +\N +\N +123 +\N +\N +\N + +-- !orc_char1_to_smallint -- +\N +\N +\N +4578 +\N +1 +2 +\N +-123 +\N +\N +\N + +-- !orc_char2_to_smallint -- +\N +\N +\N +\N +\N +3 +3 +\N +123 +\N +\N +\N + +-- !orc_varchar_to_smallint -- +\N +\N +\N +\N +\N +1 +4 +\N +123 +\N +\N +\N + +-- !orc_decimal1_to_smallint -- +\N +\N +0 +1 +1 +\N +0 +\N +123 +456 +789 +707 + +-- !orc_decimal2_to_smallint -- +\N +\N +0 +1 +2 +\N +2 +\N +\N +456 +789 +707 + +-- !orc_tinyint_to_int -- +127 +-128 +0 +1 +-2 +120 +-120 +\N +123 +-56 +21 +-61 + +-- !orc_smallint_to_int -- +32767 +-32768 +0 +1 +-2 +12345 +-12345 +\N +123 +456 +789 +707 + +-- !orc_int_to_int -- +2147483647 +-2147483648 +0 +1 +-2 +1234567890 +-1234567890 +\N +123 +456 +789 +707 + +-- !orc_string_to_int -- +\N +\N +\N +21 +127 +\N +\N +\N +123 +\N +\N +\N + +-- !orc_char1_to_int -- +\N +\N +\N +4578 +12345667 +1 +2 +\N +-123 +\N +\N +\N + +-- !orc_char2_to_int -- +\N +\N +\N +\N +\N +3 +3 +\N +123 +\N +\N +\N + +-- !orc_varchar_to_int -- +\N +\N +\N +\N +\N +1 +4 +\N +123 +\N +\N +\N + +-- !orc_decimal1_to_int -- +99999999 +-99999999 +0 +1 +1 +12345678 +0 +\N +123 +456 +789 +707 + +-- !orc_decimal2_to_int -- +\N +\N +0 +1 +2 +\N +2 +\N +123777777 +456 +789 +707 + +-- !orc_tinyint_to_bigint -- +127 +-128 +0 +1 +-2 +120 +-120 +\N +123 +-56 +21 +-61 + +-- !orc_smallint_to_bigint -- +32767 +-32768 +0 +1 +-2 +12345 +-12345 +\N +123 +456 +789 +707 + +-- !orc_int_to_bigint -- +2147483647 +-2147483648 +0 +1 +-2 +1234567890 +-1234567890 +\N +123 +456 +789 +707 + +-- !orc_bigint_to_bigint -- +9223372036854775807 +-9223372036854775808 +0 +1 +-2 +1234567890123456789 +-1234567890123456789 +\N +123 +456 +789 +707 + +-- !orc_string_to_bigint -- +\N +\N +\N +21 +127 +\N +\N +\N +123 +\N +\N +\N + +-- !orc_char1_to_bigint -- +\N +\N +\N +4578 +12345667 +1 +2 +\N +-123 +\N +\N +\N + +-- !orc_char2_to_bigint -- +\N +\N +\N +\N +\N +3 +3 +\N +123 +\N +\N +\N + +-- !orc_varchar_to_bigint -- +\N +\N +\N +\N +\N +1 +4 +\N +123 +\N +\N +\N + +-- !orc_decimal1_to_bigint -- +99999999 +-99999999 +0 +1 +1 +12345678 +0 +\N +123 +456 +789 +707 + +-- !orc_decimal2_to_bigint -- +\N +\N +0 +1 +2 +\N +2 +\N +123777777 +456 +789 +707 + +-- !orc_tinyint_to_float -- +127.0 +-128.0 +0.0 +1.0 +-2.0 +120.0 +-120.0 +\N +123.0 +-56.0 +21.0 +-61.0 + +-- !orc_smallint_to_float -- +32767.0 +-32768.0 +0.0 +1.0 +-2.0 +12345.0 +-12345.0 +\N +123.0 +456.0 +789.0 +707.0 + +-- !orc_int_to_float -- +\N +\N +0.0 +1.0 +-2.0 +\N +\N +\N +123.0 +456.0 +789.0 +707.0 + +-- !orc_bigint_to_float -- +\N +\N +0.0 +1.0 +-2.0 +\N +\N +\N +123.0 +456.0 +789.0 +707.0 + +-- !orc_float_to_float -- +3.4028235e+38 +-3.4028235e+38 +0.0 +1.1 +-2.1 +1.23E20 +-1.23E20 +\N +1.23 +4.56 +7.89 +7.07 + +-- !orc_string_to_float -- +\N +\N +\N +21.0 +127.0 +1.2 +3.4 +\N +123.0 +3.14159 +\N +1.2345 + +-- !orc_char1_to_float -- +\N +\N +\N +4578.0 +1.2345667E7 +1.0 +2.0 +\N +-123.0 +1.23E-10 +\N +1.24 + +-- !orc_char2_to_float -- +\N +\N +\N +58910.67 +345678.75 +3.0 +3.0 +\N +123.0 +1.23000003E10 +\N +-1.24 + +-- !orc_varchar_to_float -- +\N +\N +\N +354111.34 +123.456 +1.0 +4.0 +\N +123.0 +0.123 +\N +-1.674 + +-- !orc_decimal1_to_float -- +1.0E8 +-1.0E8 +0.0 +1.12 +1.14 +1.2345678E7 +0.01 +\N +123.4 +456.78 +789.01 +707.07 + +-- !orc_decimal2_to_float -- +1.0E28 +-1.0E28 +0.0 +1.1111 +2.1111 +1.2345679E27 +2.1021 +\N +1.23777776E8 +456.78778 +789.0134 +707.07886 + +-- !orc_tinyint_to_double -- +127.0 +-128.0 +0.0 +1.0 +-2.0 +120.0 +-120.0 +\N +123.0 +-56.0 +21.0 +-61.0 + +-- !orc_smallint_to_double -- +32767.0 +-32768.0 +0.0 +1.0 +-2.0 +12345.0 +-12345.0 +\N +123.0 +456.0 +789.0 +707.0 + +-- !orc_int_to_double -- +2.147483647E9 +-2.147483648E9 +0.0 +1.0 +-2.0 +1.23456789E9 +-1.23456789E9 +\N +123.0 +456.0 +789.0 +707.0 + +-- !orc_bigint_to_double -- +\N +\N +0.0 +1.0 +-2.0 +\N +\N +\N +123.0 +456.0 +789.0 +707.0 + +-- !orc_float_to_double -- +3.4028234663852886E38 +-3.4028234663852886E38 +0.0 +1.100000023841858 +-2.0999999046325684 +1.2299999965027815E20 +-1.2299999965027815E20 +\N +1.2300000190734863 +4.559999942779541 +7.889999866485596 +7.070000171661377 + +-- !orc_double_to_double -- +1.7976931348623157E308 +-1.7976931348623157E308 +0.0 +1.2 +-2.2 +1.23E40 +-1.23E40 +\N +1.23 +4.56 +7.89 +7.07 + +-- !orc_string_to_double -- +\N +\N +\N +21.0 +127.0 +1.2 +3.4 +\N +123.0 +3.14159 +\N +1.2345 + +-- !orc_char1_to_double -- +\N +\N +\N +4578.0 +1.2345667E7 +1.0 +2.0 +\N +-123.0 +1.23E-10 +\N +1.24 + +-- !orc_char2_to_double -- +\N +\N +\N +58910.67 +345678.76543 +3.0 +3.0 +\N +123.0 +1.23E10 +\N +-1.24 + +-- !orc_varchar_to_double -- +\N +\N +\N +354111.345 +123.456 +1.0 +4.0 +\N +123.0 +0.123 +\N +-1.674 + +-- !orc_decimal1_to_double -- +9.999999999E7 +-9.999999999E7 +0.0 +1.12 +1.14 +1.234567812E7 +0.01 +\N +123.4 +456.78 +789.01 +707.07 + +-- !orc_decimal2_to_double -- +1.0E28 +-1.0E28 +0.0 +1.1111 +2.1111 +1.2345678901234567E27 +2.1021 +\N +1.2377777741234E8 +456.787778 +789.013456 +707.07888 + +-- !orc_boolean_to_string -- +TRUE +FALSE +TRUE +FALSE +FALSE +FALSE +TRUE +\N +TRUE +TRUE +TRUE +TRUE + +-- !orc_tinyint_to_string -- +127 +-128 +0 +1 +-2 +120 +-120 +\N +123 +-56 +21 +-61 + +-- !orc_smallint_to_string -- +32767 +-32768 +0 +1 +-2 +12345 +-12345 +\N +123 +456 +789 +707 + +-- !orc_int_to_string -- +2147483647 +-2147483648 +0 +1 +-2 +1234567890 +-1234567890 +\N +123 +456 +789 +707 + +-- !orc_bigint_to_string -- +9223372036854775807 +-9223372036854775808 +0 +1 +-2 +1234567890123456789 +-1234567890123456789 +\N +123 +456 +789 +707 + +-- !orc_float_to_string -- +3.4028235e+38 +-3.4028235e+38 +0 +1.1 +-2.1 +1.23e+20 +-1.23e+20 +\N +1.23 +4.56 +7.89 +7.07 + +-- !orc_double_to_string -- +1.7976931348623157e+308 +-1.7976931348623157e+308 +0 +1.2 +-2.2 +1.23e+40 +-1.23e+40 +\N +1.23 +4.56 +7.89 +7.07 + +-- !orc_string_to_string -- +Sample1 +Sample2 + +21 +127 +1.2 +3.4 +\N +123 +3.14159 +true +1.2345 + +-- !orc_date_to_string -- +2023-10-22 +2020-01-01 +\N +\N +\N +2019-12-31 +2022-05-20 +\N +2023-01-01 +2023-01-01 +2023-01-01 +2023-01-01 + +-- !orc_timestamp_to_string -- +2023-10-22 12:34:56 +2020-01-01 00:00:00 +\N +\N +\N +2019-12-31 23:59:59 +2022-05-20 10:00:00 +\N +2023-01-01 00:00:00 +2023-01-01 00:00:00 +2023-01-01 00:00:00 +2023-01-01 00:00:00 + +-- !orc_decimal1_to_string -- +99999999.99 +-99999999.99 +0.00 +1.12 +1.14 +12345678.12 +0.01 +\N +123.40 +456.78 +789.01 +707.07 + +-- !orc_decimal2_to_string -- +9999999999999999999999999999.9999999999 +-9999999999999999999999999999.9999999999 +0.0000000000 +1.1111000000 +2.1111000000 +1234567890123456789012345678.1234567890 +2.1021000000 +\N +123777777.4123400000 +456.7877780000 +789.0134560000 +707.0788800000 + +-- !orc_string_to_date -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N + +-- !orc_char1_to_date -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N + +-- !orc_char2_to_date -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N + +-- !orc_varchar_to_date -- +\N +\N +\N +\N +\N +\N +\N +\N +2000-12-03 +\N +\N +\N + +-- !orc_date_to_date -- +2023-10-22 +2020-01-01 +\N +\N +\N +2019-12-31 +2022-05-20 +\N +2023-01-01 +2023-01-01 +2023-01-01 +2023-01-01 + +-- !orc_timestamp_to_date -- +2023-10-22 +2020-01-01 +\N +\N +\N +2019-12-31 +2022-05-20 +\N +2023-01-01 +2023-01-01 +2023-01-01 +2023-01-01 + +-- !orc_string_to_timestamp -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N + +-- !orc_char1_to_timestamp -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N + +-- !orc_char2_to_timestamp -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N + +-- !orc_varchar_to_timestamp -- +\N +\N +\N +\N +\N +\N +\N +\N +2000-12-03T00:00 +\N +\N +\N + +-- !orc_date_to_timestamp -- +2023-10-22T00:00 +2020-01-01T00:00 +\N +\N +\N +2019-12-31T00:00 +2022-05-20T00:00 +\N +2023-01-01T00:00 +2023-01-01T00:00 +2023-01-01T00:00 +2023-01-01T00:00 + +-- !orc_timestamp_to_timestamp -- +2023-10-22T12:34:56 +2020-01-01T00:00 +\N +\N +\N +2019-12-31T23:59:59 +2022-05-20T10:00 +\N +2023-01-01T00:00 +2023-01-01T00:00 +2023-01-01T00:00 +2023-01-01T00:00 + +-- !orc_tinyint_to_decimal1 -- +127.00000 +-128.00000 +0.00000 +1.00000 +-2.00000 +120.00000 +-120.00000 +\N +123.00000 +-56.00000 +21.00000 +-61.00000 + +-- !orc_smallint_to_decimal1 -- +32767.00000 +-32768.00000 +0.00000 +1.00000 +-2.00000 +12345.00000 +-12345.00000 +\N +123.00000 +456.00000 +789.00000 +707.00000 + +-- !orc_int_to_decimal1 -- +2147483647.00000 +-2147483648.00000 +0.00000 +1.00000 +-2.00000 +1234567890.00000 +-1234567890.00000 +\N +123.00000 +456.00000 +789.00000 +707.00000 + +-- !orc_bigint_to_decimal1 -- +\N +\N +0.00000 +1.00000 +-2.00000 +\N +\N +\N +123.00000 +456.00000 +789.00000 +707.00000 + +-- !orc_float_to_decimal1 -- +\N +\N +0.00000 +1.10000 +-2.10000 +\N +\N +\N +1.23000 +4.56000 +7.89000 +7.07000 + +-- !orc_double_to_decimal1 -- +\N +\N +0.00000 +1.20000 +-2.20000 +\N +\N +\N +1.23000 +4.56000 +7.89000 +7.07000 + +-- !orc_string_to_decimal1 -- +\N +\N +\N +21.00000 +127.00000 +1.20000 +3.40000 +\N +123.00000 +3.14159 +\N +1.23450 + +-- !orc_char1_to_decimal1 -- +\N +\N +\N +4578.00000 +12345667.00000 +1.00000 +2.00000 +\N +-123.00000 +\N +\N +1.24000 + +-- !orc_char2_to_decimal1 -- +\N +\N +\N +58910.67000 +345678.76543 +3.00000 +3.00000 +\N +123.00000 +12300000000.00000 +\N +-1.24000 + +-- !orc_varchar_to_decimal1 -- +\N +\N +\N +354111.34500 +123.45600 +1.00000 +4.00000 +\N +123.00000 +0.12300 +\N +-1.67400 + +-- !orc_decimal1_to_decimal1 -- +99999999.99000 +-99999999.99000 +0.00000 +1.12000 +1.14000 +12345678.12000 +0.01000 +\N +123.40000 +456.78000 +789.01000 +707.07000 + +-- !orc_tinyint_to_decimal2 -- +127.0 +-128.0 +0.0 +1.0 +-2.0 +120.0 +-120.0 +\N +123.0 +-56.0 +21.0 +-61.0 + +-- !orc_smallint_to_decimal2 -- +32767.0 +-32768.0 +0.0 +1.0 +-2.0 +12345.0 +-12345.0 +\N +123.0 +456.0 +789.0 +707.0 + +-- !orc_int_to_decimal2 -- +\N +\N +0.0 +1.0 +-2.0 +\N +\N +\N +123.0 +456.0 +789.0 +707.0 + +-- !orc_bigint_to_decimal2 -- +\N +\N +0.0 +1.0 +-2.0 +\N +\N +\N +123.0 +456.0 +789.0 +707.0 + +-- !orc_float_to_decimal2 -- +\N +\N +0.0 +1.1 +-2.1 +\N +\N +\N +1.2 +4.6 +7.9 +7.1 + +-- !orc_double_to_decimal2 -- +\N +\N +0.0 +1.2 +-2.2 +\N +\N +\N +1.2 +4.6 +7.9 +7.1 + +-- !orc_string_to_decimal2 -- +\N +\N +\N +21.0 +127.0 +1.2 +3.4 +\N +123.0 +\N +\N +\N + +-- !orc_char1_to_decimal2 -- +\N +\N +\N +4578.0 +\N +1.0 +2.0 +\N +-123.0 +\N +\N +\N + +-- !orc_char2_to_decimal2 -- +\N +\N +\N +\N +345678.8 +3.0 +3.0 +\N +123.0 +\N +\N +\N + +-- !orc_varchar_to_decimal2 -- +\N +\N +\N +354111.3 +\N +1.0 +4.0 +\N +123.0 +\N +\N +\N + diff --git a/regression-test/data/external_table_p0/hive/test_hive_schema_change_parquet.out b/regression-test/data/external_table_p0/hive/test_hive_schema_change_parquet.out new file mode 100644 index 00000000000000..8645792bfa837e --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_hive_schema_change_parquet.out @@ -0,0 +1,1653 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !parquet_boolean_to_boolean -- +true +false +true +false +false +false +true +\N +true +true +true +true + +-- !parquet_string_to_boolean -- +true +true +false +true +true +true +true +\N +true +true +true +true + +-- !parquet_char1_to_boolean -- +true +true +false +true +true +true +true +\N +true +true +false +true + +-- !parquet_char2_to_boolean -- +true +true +false +true +true +true +true +\N +true +true +true +true + +-- !parquet_varchar_to_boolean -- +true +true +false +true +true +true +true +\N +true +true +false +true + +-- !parquet_tinyint_to_smallint -- +127 +-128 +0 +1 +-2 +120 +-120 +\N +123 +-56 +21 +-61 + +-- !parquet_smallint_to_smallint -- +32767 +-32768 +0 +1 +-2 +12345 +-12345 +\N +123 +456 +789 +707 + +-- !parquet_string_to_smallint -- +\N +\N +\N +21 +127 +\N +\N +\N +123 +\N +\N +\N + +-- !parquet_char1_to_smallint -- +\N +\N +\N +4578 +\N +1 +2 +\N +-123 +\N +\N +\N + +-- !parquet_char2_to_smallint -- +\N +\N +\N +\N +\N +3 +3 +\N +123 +\N +\N +\N + +-- !parquet_varchar_to_smallint -- +\N +\N +\N +\N +\N +1 +4 +\N +123 +\N +\N +\N + +-- !parquet_decimal1_to_smallint -- +\N +\N +0 +1 +1 +\N +0 +\N +123 +456 +789 +707 + +-- !parquet_decimal2_to_smallint -- +\N +\N +0 +1 +2 +\N +2 +\N +\N +456 +789 +707 + +-- !parquet_tinyint_to_tinyint -- +127 +-128 +0 +1 +-2 +120 +-120 +\N +123 +-56 +21 +-61 + +-- !parquet_string_to_tinyint -- +\N +\N +\N +21 +127 +\N +\N +\N +123 +\N +\N +\N + +-- !parquet_char1_to_tinyint -- +\N +\N +\N +\N +\N +1 +2 +\N +-123 +\N +\N +\N + +-- !parquet_char2_to_tinyint -- +\N +\N +\N +\N +\N +3 +3 +\N +123 +\N +\N +\N + +-- !parquet_varchar_to_tinyint -- +\N +\N +\N +\N +\N +1 +4 +\N +123 +\N +\N +\N + +-- !parquet_decimal1_to_tinyint -- +\N +\N +0 +1 +1 +\N +0 +\N +123 +\N +\N +\N + +-- !parquet_decimal2_to_tinyint -- +\N +\N +0 +1 +2 +\N +2 +\N +\N +\N +\N +\N + +-- !parquet_tinyint_to_smallint -- +127 +-128 +0 +1 +-2 +120 +-120 +\N +123 +-56 +21 +-61 + +-- !parquet_smallint_to_smallint -- +32767 +-32768 +0 +1 +-2 +12345 +-12345 +\N +123 +456 +789 +707 + +-- !parquet_string_to_smallint -- +\N +\N +\N +21 +127 +\N +\N +\N +123 +\N +\N +\N + +-- !parquet_char1_to_smallint -- +\N +\N +\N +4578 +\N +1 +2 +\N +-123 +\N +\N +\N + +-- !parquet_char2_to_smallint -- +\N +\N +\N +\N +\N +3 +3 +\N +123 +\N +\N +\N + +-- !parquet_varchar_to_smallint -- +\N +\N +\N +\N +\N +1 +4 +\N +123 +\N +\N +\N + +-- !parquet_decimal1_to_smallint -- +\N +\N +0 +1 +1 +\N +0 +\N +123 +456 +789 +707 + +-- !parquet_decimal2_to_smallint -- +\N +\N +0 +1 +2 +\N +2 +\N +\N +456 +789 +707 + +-- !parquet_tinyint_to_int -- +127 +-128 +0 +1 +-2 +120 +-120 +\N +123 +-56 +21 +-61 + +-- !parquet_smallint_to_int -- +32767 +-32768 +0 +1 +-2 +12345 +-12345 +\N +123 +456 +789 +707 + +-- !parquet_int_to_int -- +2147483647 +-2147483648 +0 +1 +-2 +1234567890 +-1234567890 +\N +123 +456 +789 +707 + +-- !parquet_string_to_int -- +\N +\N +\N +21 +127 +\N +\N +\N +123 +\N +\N +\N + +-- !parquet_char1_to_int -- +\N +\N +\N +4578 +12345667 +1 +2 +\N +-123 +\N +\N +\N + +-- !parquet_char2_to_int -- +\N +\N +\N +\N +\N +3 +3 +\N +123 +\N +\N +\N + +-- !parquet_varchar_to_int -- +\N +\N +\N +\N +\N +1 +4 +\N +123 +\N +\N +\N + +-- !parquet_decimal1_to_int -- +99999999 +-99999999 +0 +1 +1 +12345678 +0 +\N +123 +456 +789 +707 + +-- !parquet_decimal2_to_int -- +\N +\N +0 +1 +2 +\N +2 +\N +123777777 +456 +789 +707 + +-- !parquet_tinyint_to_bigint -- +127 +-128 +0 +1 +-2 +120 +-120 +\N +123 +-56 +21 +-61 + +-- !parquet_smallint_to_bigint -- +32767 +-32768 +0 +1 +-2 +12345 +-12345 +\N +123 +456 +789 +707 + +-- !parquet_int_to_bigint -- +2147483647 +-2147483648 +0 +1 +-2 +1234567890 +-1234567890 +\N +123 +456 +789 +707 + +-- !parquet_bigint_to_bigint -- +9223372036854775807 +-9223372036854775808 +0 +1 +-2 +1234567890123456789 +-1234567890123456789 +\N +123 +456 +789 +707 + +-- !parquet_string_to_bigint -- +\N +\N +\N +21 +127 +\N +\N +\N +123 +\N +\N +\N + +-- !parquet_char1_to_bigint -- +\N +\N +\N +4578 +12345667 +1 +2 +\N +-123 +\N +\N +\N + +-- !parquet_char2_to_bigint -- +\N +\N +\N +\N +\N +3 +3 +\N +123 +\N +\N +\N + +-- !parquet_varchar_to_bigint -- +\N +\N +\N +\N +\N +1 +4 +\N +123 +\N +\N +\N + +-- !parquet_decimal1_to_bigint -- +99999999 +-99999999 +0 +1 +1 +12345678 +0 +\N +123 +456 +789 +707 + +-- !parquet_decimal2_to_bigint -- +\N +\N +0 +1 +2 +\N +2 +\N +123777777 +456 +789 +707 + +-- !parquet_tinyint_to_float -- +127.0 +-128.0 +0.0 +1.0 +-2.0 +120.0 +-120.0 +\N +123.0 +-56.0 +21.0 +-61.0 + +-- !parquet_smallint_to_float -- +32767.0 +-32768.0 +0.0 +1.0 +-2.0 +12345.0 +-12345.0 +\N +123.0 +456.0 +789.0 +707.0 + +-- !parquet_int_to_float -- +\N +\N +0.0 +1.0 +-2.0 +\N +\N +\N +123.0 +456.0 +789.0 +707.0 + +-- !parquet_bigint_to_float -- +\N +\N +0.0 +1.0 +-2.0 +\N +\N +\N +123.0 +456.0 +789.0 +707.0 + +-- !parquet_float_to_float -- +3.4028235e+38 +-3.4028235e+38 +0.0 +1.1 +-2.1 +1.23E20 +-1.23E20 +\N +1.23 +4.56 +7.89 +7.07 + +-- !parquet_string_to_float -- +\N +\N +\N +21.0 +127.0 +1.2 +3.4 +\N +123.0 +3.14159 +\N +1.2345 + +-- !parquet_char1_to_float -- +\N +\N +\N +4578.0 +1.2345667E7 +1.0 +2.0 +\N +-123.0 +1.23E-10 +\N +1.24 + +-- !parquet_char2_to_float -- +\N +\N +\N +58910.67 +345678.75 +3.0 +3.0 +\N +123.0 +1.23000003E10 +\N +-1.24 + +-- !parquet_varchar_to_float -- +\N +\N +\N +354111.34 +123.456 +1.0 +4.0 +\N +123.0 +0.123 +\N +-1.674 + +-- !parquet_decimal1_to_float -- +1.0E8 +-1.0E8 +0.0 +1.12 +1.14 +1.2345678E7 +0.01 +\N +123.4 +456.78 +789.01 +707.07 + +-- !parquet_decimal2_to_float -- +1.0E28 +-1.0E28 +0.0 +1.1111 +2.1111 +1.2345679E27 +2.1021 +\N +1.23777776E8 +456.78778 +789.0134 +707.07886 + +-- !parquet_tinyint_to_double -- +127.0 +-128.0 +0.0 +1.0 +-2.0 +120.0 +-120.0 +\N +123.0 +-56.0 +21.0 +-61.0 + +-- !parquet_smallint_to_double -- +32767.0 +-32768.0 +0.0 +1.0 +-2.0 +12345.0 +-12345.0 +\N +123.0 +456.0 +789.0 +707.0 + +-- !parquet_int_to_double -- +2.147483647E9 +-2.147483648E9 +0.0 +1.0 +-2.0 +1.23456789E9 +-1.23456789E9 +\N +123.0 +456.0 +789.0 +707.0 + +-- !parquet_bigint_to_double -- +\N +\N +0.0 +1.0 +-2.0 +\N +\N +\N +123.0 +456.0 +789.0 +707.0 + +-- !parquet_float_to_double -- +3.4028234663852886E38 +-3.4028234663852886E38 +0.0 +1.100000023841858 +-2.0999999046325684 +1.2299999965027815E20 +-1.2299999965027815E20 +\N +1.2300000190734863 +4.559999942779541 +7.889999866485596 +7.070000171661377 + +-- !parquet_double_to_double -- +1.7976931348623157E308 +-1.7976931348623157E308 +0.0 +1.2 +-2.2 +1.23E40 +-1.23E40 +\N +1.23 +4.56 +7.89 +7.07 + +-- !parquet_string_to_double -- +\N +\N +\N +21.0 +127.0 +1.2 +3.4 +\N +123.0 +3.14159 +\N +1.2345 + +-- !parquet_char1_to_double -- +\N +\N +\N +4578.0 +1.2345667E7 +1.0 +2.0 +\N +-123.0 +1.23E-10 +\N +1.24 + +-- !parquet_char2_to_double -- +\N +\N +\N +58910.67 +345678.76543 +3.0 +3.0 +\N +123.0 +1.23E10 +\N +-1.24 + +-- !parquet_varchar_to_double -- +\N +\N +\N +354111.345 +123.456 +1.0 +4.0 +\N +123.0 +0.123 +\N +-1.674 + +-- !parquet_decimal1_to_double -- +9.999999999E7 +-9.999999999E7 +0.0 +1.12 +1.14 +1.234567812E7 +0.01 +\N +123.4 +456.78 +789.01 +707.07 + +-- !parquet_decimal2_to_double -- +1.0E28 +-1.0E28 +0.0 +1.1111 +2.1111 +1.2345678901234567E27 +2.1021 +\N +1.2377777741234E8 +456.787778 +789.013456 +707.07888 + +-- !parquet_boolean_to_string -- +TRUE +FALSE +TRUE +FALSE +FALSE +FALSE +TRUE +\N +TRUE +TRUE +TRUE +TRUE + +-- !parquet_tinyint_to_string -- +127 +-128 +0 +1 +-2 +120 +-120 +\N +123 +-56 +21 +-61 + +-- !parquet_smallint_to_string -- +32767 +-32768 +0 +1 +-2 +12345 +-12345 +\N +123 +456 +789 +707 + +-- !parquet_int_to_string -- +2147483647 +-2147483648 +0 +1 +-2 +1234567890 +-1234567890 +\N +123 +456 +789 +707 + +-- !parquet_bigint_to_string -- +9223372036854775807 +-9223372036854775808 +0 +1 +-2 +1234567890123456789 +-1234567890123456789 +\N +123 +456 +789 +707 + +-- !parquet_float_to_string -- +3.4028235e+38 +-3.4028235e+38 +0 +1.1 +-2.1 +1.23e+20 +-1.23e+20 +\N +1.23 +4.56 +7.89 +7.07 + +-- !parquet_double_to_string -- +1.7976931348623157e+308 +-1.7976931348623157e+308 +0 +1.2 +-2.2 +1.23e+40 +-1.23e+40 +\N +1.23 +4.56 +7.89 +7.07 + +-- !parquet_string_to_string -- +Sample1 +Sample2 + +21 +127 +1.2 +3.4 +\N +123 +3.14159 +true +1.2345 + +-- !parquet_char1_to_string -- +CHAR1CHAR1 +CHAR1CHAR1 + +4578 +12345667 +1 +2 +\N +-123 +1.23e-10 +false +1.24 + +-- !parquet_char2_to_string -- +CHAR1CHAR1CHAR1CHAR1 +CHAR1CHAR1CHAR1CHAR1 + +58910.67 +345678.76543 +3 +3 +\N ++123 +1.23E+10 +TRUE +-1.24 + +-- !parquet_varchar_to_string -- +VARCHAR123 +VARCHAR123 + +354111.345 +123.456 +1 +4 +\N +00123 +.123 +FALSE +-1.674 + +-- !parquet_date_to_string -- +2023-10-22 +2020-01-01 +\N +\N +\N +2019-12-31 +2022-05-20 +\N +2023-01-01 +2023-01-01 +2023-01-01 +2023-01-01 + +-- !parquet_timestamp_to_string -- +2023-10-22 12:34:56 +2020-01-01 00:00:00 +\N +\N +\N +2019-12-31 23:59:59 +2022-05-20 10:00:00 +\N +2023-01-01 00:00:00 +2023-01-01 00:00:00 +2023-01-01 00:00:00 +2023-01-01 00:00:00 + +-- !parquet_decimal1_to_string -- +99999999.99 +-99999999.99 +0.00 +1.12 +1.14 +12345678.12 +0.01 +\N +123.40 +456.78 +789.01 +707.07 + +-- !parquet_decimal2_to_string -- +9999999999999999999999999999.9999999999 +-9999999999999999999999999999.9999999999 +0.0000000000 +1.1111000000 +2.1111000000 +1234567890123456789012345678.1234567890 +2.1021000000 +\N +123777777.4123400000 +456.7877780000 +789.0134560000 +707.0788800000 + +-- !parquet_string_to_date -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N + +-- !parquet_char1_to_date -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N + +-- !parquet_char2_to_date -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N + +-- !parquet_varchar_to_date -- +\N +\N +\N +\N +\N +\N +\N +\N +2000-12-03 +\N +\N +\N + +-- !parquet_date_to_date -- +2023-10-22 +2020-01-01 +\N +\N +\N +2019-12-31 +2022-05-20 +\N +2023-01-01 +2023-01-01 +2023-01-01 +2023-01-01 + +-- !parquet_timestamp_to_date -- +2023-10-22 +2020-01-01 +\N +\N +\N +2019-12-31 +2022-05-20 +\N +2023-01-01 +2023-01-01 +2023-01-01 +2023-01-01 + +-- !parquet_string_to_timestamp -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N + +-- !parquet_char1_to_timestamp -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N + +-- !parquet_char2_to_timestamp -- +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N +\N + +-- !parquet_varchar_to_timestamp -- +\N +\N +\N +\N +\N +\N +\N +\N +2000-12-03T00:00 +\N +\N +\N + +-- !parquet_date_to_timestamp -- +2023-10-22T00:00 +2020-01-01T00:00 +\N +\N +\N +2019-12-31T00:00 +2022-05-20T00:00 +\N +2023-01-01T00:00 +2023-01-01T00:00 +2023-01-01T00:00 +2023-01-01T00:00 + +-- !parquet_timestamp_to_timestamp -- +2023-10-22T12:34:56 +2020-01-01T00:00 +\N +\N +\N +2019-12-31T23:59:59 +2022-05-20T10:00 +\N +2023-01-01T00:00 +2023-01-01T00:00 +2023-01-01T00:00 +2023-01-01T00:00 + +-- !parquet_tinyint_to_decimal1 -- +127.00000 +-128.00000 +0.00000 +1.00000 +-2.00000 +120.00000 +-120.00000 +\N +123.00000 +-56.00000 +21.00000 +-61.00000 + +-- !parquet_smallint_to_decimal1 -- +32767.00000 +-32768.00000 +0.00000 +1.00000 +-2.00000 +12345.00000 +-12345.00000 +\N +123.00000 +456.00000 +789.00000 +707.00000 + +-- !parquet_int_to_decimal1 -- +2147483647.00000 +-2147483648.00000 +0.00000 +1.00000 +-2.00000 +1234567890.00000 +-1234567890.00000 +\N +123.00000 +456.00000 +789.00000 +707.00000 + +-- !parquet_bigint_to_decimal1 -- +\N +\N +0.00000 +1.00000 +-2.00000 +\N +\N +\N +123.00000 +456.00000 +789.00000 +707.00000 + +-- !parquet_float_to_decimal1 -- +\N +\N +0.00000 +1.10000 +-2.10000 +\N +\N +\N +1.23000 +4.56000 +7.89000 +7.07000 + +-- !parquet_double_to_decimal1 -- +\N +\N +0.00000 +1.20000 +-2.20000 +\N +\N +\N +1.23000 +4.56000 +7.89000 +7.07000 + +-- !parquet_string_to_decimal1 -- +\N +\N +\N +21.00000 +127.00000 +1.20000 +3.40000 +\N +123.00000 +3.14159 +\N +1.23450 + +-- !parquet_char1_to_decimal1 -- +\N +\N +\N +4578.00000 +12345667.00000 +1.00000 +2.00000 +\N +-123.00000 +\N +\N +1.24000 + +-- !parquet_char2_to_decimal1 -- +\N +\N +\N +58910.67000 +345678.76543 +3.00000 +3.00000 +\N +123.00000 +12300000000.00000 +\N +-1.24000 + +-- !parquet_varchar_to_decimal1 -- +\N +\N +\N +354111.34500 +123.45600 +1.00000 +4.00000 +\N +123.00000 +0.12300 +\N +-1.67400 + +-- !parquet_decimal1_to_decimal1 -- +99999999.99000 +-99999999.99000 +0.00000 +1.12000 +1.14000 +12345678.12000 +0.01000 +\N +123.40000 +456.78000 +789.01000 +707.07000 + +-- !parquet_tinyint_to_decimal2 -- +127.0 +-128.0 +0.0 +1.0 +-2.0 +120.0 +-120.0 +\N +123.0 +-56.0 +21.0 +-61.0 + +-- !parquet_smallint_to_decimal2 -- +32767.0 +-32768.0 +0.0 +1.0 +-2.0 +12345.0 +-12345.0 +\N +123.0 +456.0 +789.0 +707.0 + +-- !parquet_int_to_decimal2 -- +\N +\N +0.0 +1.0 +-2.0 +\N +\N +\N +123.0 +456.0 +789.0 +707.0 + +-- !parquet_bigint_to_decimal2 -- +\N +\N +0.0 +1.0 +-2.0 +\N +\N +\N +123.0 +456.0 +789.0 +707.0 + +-- !parquet_float_to_decimal2 -- +\N +\N +0.0 +1.1 +-2.1 +\N +\N +\N +1.2 +4.6 +7.9 +7.1 + +-- !parquet_double_to_decimal2 -- +\N +\N +0.0 +1.2 +-2.2 +\N +\N +\N +1.2 +4.6 +7.9 +7.1 + +-- !parquet_string_to_decimal2 -- +\N +\N +\N +21.0 +127.0 +1.2 +3.4 +\N +123.0 +\N +\N +\N + +-- !parquet_char1_to_decimal2 -- +\N +\N +\N +4578.0 +\N +1.0 +2.0 +\N +-123.0 +\N +\N +\N + +-- !parquet_char2_to_decimal2 -- +\N +\N +\N +\N +345678.8 +3.0 +3.0 +\N +123.0 +\N +\N +\N + +-- !parquet_varchar_to_decimal2 -- +\N +\N +\N +354111.3 +\N +1.0 +4.0 +\N +123.0 +\N +\N +\N + diff --git a/regression-test/suites/external_table_p0/hive/test_hive_schema_change_orc.groovy b/regression-test/suites/external_table_p0/hive/test_hive_schema_change_orc.groovy new file mode 100644 index 00000000000000..2ac4e00ef7ff10 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_hive_schema_change_orc.groovy @@ -0,0 +1,1034 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_schema_change_orc", "p0,external,hive,external_docker,external_docker_hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + for (String hivePrefix : ["hive3"]) { + setHivePrefix(hivePrefix) + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String hmsPort = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") + String catalog_name = "test_hive_schema_change_orc" + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hadoop.username' = 'hadoop', + 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}' + ); + """ + sql """ switch ${catalog_name} """ + + sql """ use `schema_change` """ + + try { + qt_orc_boolean_to_boolean """ select bool_col from orc_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_orc_tinyint_to_boolean """ select tinyint_col from orc_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_orc_smallint_to_boolean """ select smallint_col from orc_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_orc_int_to_boolean """ select int_col from orc_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_orc_bigint_to_boolean """ select bigint_col from orc_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_orc_float_to_boolean """ select float_col from orc_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_orc_double_to_boolean """ select double_col from orc_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_orc_string_to_boolean """ select string_col from orc_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char1_to_boolean """ select char1_col from orc_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char2_to_boolean """ select char2_col from orc_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_orc_varchar_to_boolean """ select varchar_col from orc_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_orc_date_to_boolean """ select date_col from orc_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_orc_timestamp_to_boolean """ select timestamp_col from orc_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal1_to_boolean """ select decimal1_col from orc_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal2_to_boolean """ select decimal2_col from orc_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + + try { + qt_orc_boolean_to_smallint """ select bool_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_tinyint_to_smallint """ select tinyint_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_smallint_to_smallint """ select smallint_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_int_to_smallint """ select int_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_bigint_to_smallint """ select bigint_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_float_to_smallint """ select float_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_double_to_smallint """ select double_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_string_to_smallint """ select string_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char1_to_smallint """ select char1_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char2_to_smallint """ select char2_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_varchar_to_smallint """ select varchar_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_date_to_smallint """ select date_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_timestamp_to_smallint """ select timestamp_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal1_to_smallint """ select decimal1_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal2_to_smallint """ select decimal2_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + + try { + qt_orc_boolean_to_tinyint """ select bool_col from orc_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_tinyint_to_tinyint """ select tinyint_col from orc_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_smallint_to_tinyint """ select smallint_col from orc_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_int_to_tinyint """ select int_col from orc_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_bigint_to_tinyint """ select bigint_col from orc_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_float_to_tinyint """ select float_col from orc_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_double_to_tinyint """ select double_col from orc_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_string_to_tinyint """ select string_col from orc_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char1_to_tinyint """ select char1_col from orc_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char2_to_tinyint """ select char2_col from orc_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_varchar_to_tinyint """ select varchar_col from orc_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_date_to_tinyint """ select date_col from orc_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_timestamp_to_tinyint """ select timestamp_col from orc_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal1_to_tinyint """ select decimal1_col from orc_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal2_to_tinyint """ select decimal2_col from orc_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + + try { + qt_orc_boolean_to_smallint """ select bool_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_tinyint_to_smallint """ select tinyint_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_smallint_to_smallint """ select smallint_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_int_to_smallint """ select int_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_bigint_to_smallint """ select bigint_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_float_to_smallint """ select float_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_double_to_smallint """ select double_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_string_to_smallint """ select string_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char1_to_smallint """ select char1_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char2_to_smallint """ select char2_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_varchar_to_smallint """ select varchar_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_date_to_smallint """ select date_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_timestamp_to_smallint """ select timestamp_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal1_to_smallint """ select decimal1_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal2_to_smallint """ select decimal2_col from orc_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + + try { + qt_orc_boolean_to_int """ select bool_col from orc_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_orc_tinyint_to_int """ select tinyint_col from orc_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_orc_smallint_to_int """ select smallint_col from orc_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_orc_int_to_int """ select int_col from orc_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_orc_bigint_to_int """ select bigint_col from orc_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_orc_float_to_int """ select float_col from orc_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_orc_double_to_int """ select double_col from orc_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_orc_string_to_int """ select string_col from orc_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char1_to_int """ select char1_col from orc_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char2_to_int """ select char2_col from orc_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_orc_varchar_to_int """ select varchar_col from orc_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_orc_date_to_int """ select date_col from orc_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_orc_timestamp_to_int """ select timestamp_col from orc_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal1_to_int """ select decimal1_col from orc_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal2_to_int """ select decimal2_col from orc_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + + + try { + qt_orc_boolean_to_bigint """ select bool_col from orc_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_tinyint_to_bigint """ select tinyint_col from orc_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_smallint_to_bigint """ select smallint_col from orc_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_int_to_bigint """ select int_col from orc_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_bigint_to_bigint """ select bigint_col from orc_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_float_to_bigint """ select float_col from orc_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_double_to_bigint """ select double_col from orc_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_string_to_bigint """ select string_col from orc_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char1_to_bigint """ select char1_col from orc_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char2_to_bigint """ select char2_col from orc_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_varchar_to_bigint """ select varchar_col from orc_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_date_to_bigint """ select date_col from orc_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_timestamp_to_bigint """ select timestamp_col from orc_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal1_to_bigint """ select decimal1_col from orc_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal2_to_bigint """ select decimal2_col from orc_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + + try { + qt_orc_boolean_to_float """ select bool_col from orc_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_orc_tinyint_to_float """ select tinyint_col from orc_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_orc_smallint_to_float """ select smallint_col from orc_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_orc_int_to_float """ select int_col from orc_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_orc_bigint_to_float """ select bigint_col from orc_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_orc_float_to_float """ select float_col from orc_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_orc_double_to_float """ select double_col from orc_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_orc_string_to_float """ select string_col from orc_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char1_to_float """ select char1_col from orc_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char2_to_float """ select char2_col from orc_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_orc_varchar_to_float """ select varchar_col from orc_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_orc_date_to_float """ select date_col from orc_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_orc_timestamp_to_float """ select timestamp_col from orc_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal1_to_float """ select decimal1_col from orc_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal2_to_float """ select decimal2_col from orc_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + + try { + qt_orc_boolean_to_double """ select bool_col from orc_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_orc_tinyint_to_double """ select tinyint_col from orc_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_orc_smallint_to_double """ select smallint_col from orc_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_orc_int_to_double """ select int_col from orc_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_orc_bigint_to_double """ select bigint_col from orc_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_orc_float_to_double """ select float_col from orc_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_orc_double_to_double """ select double_col from orc_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_orc_string_to_double """ select string_col from orc_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char1_to_double """ select char1_col from orc_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char2_to_double """ select char2_col from orc_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_orc_varchar_to_double """ select varchar_col from orc_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_orc_date_to_double """ select date_col from orc_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_orc_timestamp_to_double """ select timestamp_col from orc_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal1_to_double """ select decimal1_col from orc_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal2_to_double """ select decimal2_col from orc_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + + try { + qt_orc_boolean_to_string """ select bool_col from orc_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_orc_tinyint_to_string """ select tinyint_col from orc_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_orc_smallint_to_string """ select smallint_col from orc_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_orc_int_to_string """ select int_col from orc_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_orc_bigint_to_string """ select bigint_col from orc_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_orc_float_to_string """ select float_col from orc_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_orc_double_to_string """ select double_col from orc_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_orc_string_to_string """ select string_col from orc_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char1_to_string """ select char1_col from orc_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char2_to_string """ select char2_col from orc_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_orc_varchar_to_string """ select varchar_col from orc_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_orc_date_to_string """ select date_col from orc_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_orc_timestamp_to_string """ select timestamp_col from orc_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal1_to_string """ select decimal1_col from orc_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal2_to_string """ select decimal2_col from orc_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + + try { + qt_orc_boolean_to_date """ select bool_col from orc_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_orc_tinyint_to_date """ select tinyint_col from orc_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_orc_smallint_to_date """ select smallint_col from orc_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_orc_int_to_date """ select int_col from orc_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_orc_bigint_to_date """ select bigint_col from orc_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_orc_float_to_date """ select float_col from orc_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_orc_double_to_date """ select double_col from orc_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_orc_string_to_date """ select string_col from orc_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char1_to_date """ select char1_col from orc_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char2_to_date """ select char2_col from orc_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_orc_varchar_to_date """ select varchar_col from orc_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_orc_date_to_date """ select date_col from orc_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_orc_timestamp_to_date """ select timestamp_col from orc_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal1_to_date """ select decimal1_col from orc_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal2_to_date """ select decimal2_col from orc_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + + + try { + qt_orc_boolean_to_timestamp """ select bool_col from orc_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_orc_tinyint_to_timestamp """ select tinyint_col from orc_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_orc_smallint_to_timestamp """ select smallint_col from orc_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_orc_int_to_timestamp """ select int_col from orc_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_orc_bigint_to_timestamp """ select bigint_col from orc_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_orc_float_to_timestamp """ select float_col from orc_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_orc_double_to_timestamp """ select double_col from orc_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_orc_string_to_timestamp """ select string_col from orc_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char1_to_timestamp """ select char1_col from orc_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char2_to_timestamp """ select char2_col from orc_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_orc_varchar_to_timestamp """ select varchar_col from orc_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_orc_date_to_timestamp """ select date_col from orc_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_orc_timestamp_to_timestamp """ select timestamp_col from orc_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal1_to_timestamp """ select decimal1_col from orc_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal2_to_timestamp """ select decimal2_col from orc_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + + try { + qt_orc_boolean_to_decimal1 """ select bool_col from orc_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_tinyint_to_decimal1 """ select tinyint_col from orc_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_smallint_to_decimal1 """ select smallint_col from orc_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_int_to_decimal1 """ select int_col from orc_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_bigint_to_decimal1 """ select bigint_col from orc_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_float_to_decimal1 """ select float_col from orc_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_double_to_decimal1 """ select double_col from orc_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_string_to_decimal1 """ select string_col from orc_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char1_to_decimal1 """ select char1_col from orc_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char2_to_decimal1 """ select char2_col from orc_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_varchar_to_decimal1 """ select varchar_col from orc_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_date_to_decimal1 """ select date_col from orc_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_timestamp_to_decimal1 """ select timestamp_col from orc_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal1_to_decimal1 """ select decimal1_col from orc_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal2_to_decimal1 """ select decimal2_col from orc_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + + + try { + qt_orc_boolean_to_decimal2 """ select bool_col from orc_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_tinyint_to_decimal2 """ select tinyint_col from orc_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_smallint_to_decimal2 """ select smallint_col from orc_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_int_to_decimal2 """ select int_col from orc_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_bigint_to_decimal2 """ select bigint_col from orc_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_float_to_decimal2 """ select float_col from orc_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_double_to_decimal2 """ select double_col from orc_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_string_to_decimal2 """ select string_col from orc_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char1_to_decimal2 """ select char1_col from orc_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_char2_to_decimal2 """ select char2_col from orc_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_varchar_to_decimal2 """ select varchar_col from orc_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_date_to_decimal2 """ select date_col from orc_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_timestamp_to_decimal2 """ select timestamp_col from orc_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal1_to_decimal2 """ select decimal1_col from orc_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_orc_decimal2_to_decimal2 """ select decimal2_col from orc_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + + sql """ drop catalog ${catalog_name} """ + } + } +} diff --git a/regression-test/suites/external_table_p0/hive/test_hive_schema_change_parquet.groovy b/regression-test/suites/external_table_p0/hive/test_hive_schema_change_parquet.groovy new file mode 100644 index 00000000000000..8979ef684af534 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_hive_schema_change_parquet.groovy @@ -0,0 +1,1034 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_schema_change_parquet", "p0,external,hive,external_docker,external_docker_hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + for (String hivePrefix : ["hive3"]) { + setHivePrefix(hivePrefix) + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String hmsPort = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") + String catalog_name = "test_hive_schema_change_parquet" + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog if not exists ${catalog_name} properties ( + 'type'='hms', + 'hadoop.username' = 'hadoop', + 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}' + ); + """ + sql """ switch ${catalog_name} """ + + sql """ use `schema_change` """ + + try { + qt_parquet_boolean_to_boolean """ select bool_col from parquet_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_tinyint_to_boolean """ select tinyint_col from parquet_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_smallint_to_boolean """ select smallint_col from parquet_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_int_to_boolean """ select int_col from parquet_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_bigint_to_boolean """ select bigint_col from parquet_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_float_to_boolean """ select float_col from parquet_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_double_to_boolean """ select double_col from parquet_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_string_to_boolean """ select string_col from parquet_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char1_to_boolean """ select char1_col from parquet_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char2_to_boolean """ select char2_col from parquet_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_varchar_to_boolean """ select varchar_col from parquet_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_date_to_boolean """ select date_col from parquet_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_timestamp_to_boolean """ select timestamp_col from parquet_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal1_to_boolean """ select decimal1_col from parquet_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal2_to_boolean """ select decimal2_col from parquet_primitive_types_to_boolean order by id """ + } catch (Exception e) { + } + + + try { + qt_parquet_boolean_to_smallint """ select bool_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_tinyint_to_smallint """ select tinyint_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_smallint_to_smallint """ select smallint_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_int_to_smallint """ select int_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_bigint_to_smallint """ select bigint_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_float_to_smallint """ select float_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_double_to_smallint """ select double_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_string_to_smallint """ select string_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char1_to_smallint """ select char1_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char2_to_smallint """ select char2_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_varchar_to_smallint """ select varchar_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_date_to_smallint """ select date_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_timestamp_to_smallint """ select timestamp_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal1_to_smallint """ select decimal1_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal2_to_smallint """ select decimal2_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + + try { + qt_parquet_boolean_to_tinyint """ select bool_col from parquet_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_tinyint_to_tinyint """ select tinyint_col from parquet_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_smallint_to_tinyint """ select smallint_col from parquet_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_int_to_tinyint """ select int_col from parquet_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_bigint_to_tinyint """ select bigint_col from parquet_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_float_to_tinyint """ select float_col from parquet_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_double_to_tinyint """ select double_col from parquet_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_string_to_tinyint """ select string_col from parquet_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char1_to_tinyint """ select char1_col from parquet_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char2_to_tinyint """ select char2_col from parquet_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_varchar_to_tinyint """ select varchar_col from parquet_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_date_to_tinyint """ select date_col from parquet_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_timestamp_to_tinyint """ select timestamp_col from parquet_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal1_to_tinyint """ select decimal1_col from parquet_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal2_to_tinyint """ select decimal2_col from parquet_primitive_types_to_tinyint order by id """ + } catch (Exception e) { + } + + + try { + qt_parquet_boolean_to_smallint """ select bool_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_tinyint_to_smallint """ select tinyint_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_smallint_to_smallint """ select smallint_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_int_to_smallint """ select int_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_bigint_to_smallint """ select bigint_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_float_to_smallint """ select float_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_double_to_smallint """ select double_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_string_to_smallint """ select string_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char1_to_smallint """ select char1_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char2_to_smallint """ select char2_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_varchar_to_smallint """ select varchar_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_date_to_smallint """ select date_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_timestamp_to_smallint """ select timestamp_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal1_to_smallint """ select decimal1_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal2_to_smallint """ select decimal2_col from parquet_primitive_types_to_smallint order by id """ + } catch (Exception e) { + } + + + try { + qt_parquet_boolean_to_int """ select bool_col from parquet_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_tinyint_to_int """ select tinyint_col from parquet_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_smallint_to_int """ select smallint_col from parquet_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_int_to_int """ select int_col from parquet_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_bigint_to_int """ select bigint_col from parquet_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_float_to_int """ select float_col from parquet_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_double_to_int """ select double_col from parquet_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_string_to_int """ select string_col from parquet_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char1_to_int """ select char1_col from parquet_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char2_to_int """ select char2_col from parquet_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_varchar_to_int """ select varchar_col from parquet_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_date_to_int """ select date_col from parquet_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_timestamp_to_int """ select timestamp_col from parquet_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal1_to_int """ select decimal1_col from parquet_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal2_to_int """ select decimal2_col from parquet_primitive_types_to_int order by id """ + } catch (Exception e) { + } + + + + try { + qt_parquet_boolean_to_bigint """ select bool_col from parquet_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_tinyint_to_bigint """ select tinyint_col from parquet_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_smallint_to_bigint """ select smallint_col from parquet_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_int_to_bigint """ select int_col from parquet_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_bigint_to_bigint """ select bigint_col from parquet_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_float_to_bigint """ select float_col from parquet_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_double_to_bigint """ select double_col from parquet_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_string_to_bigint """ select string_col from parquet_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char1_to_bigint """ select char1_col from parquet_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char2_to_bigint """ select char2_col from parquet_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_varchar_to_bigint """ select varchar_col from parquet_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_date_to_bigint """ select date_col from parquet_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_timestamp_to_bigint """ select timestamp_col from parquet_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal1_to_bigint """ select decimal1_col from parquet_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal2_to_bigint """ select decimal2_col from parquet_primitive_types_to_bigint order by id """ + } catch (Exception e) { + } + + + try { + qt_parquet_boolean_to_float """ select bool_col from parquet_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_tinyint_to_float """ select tinyint_col from parquet_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_smallint_to_float """ select smallint_col from parquet_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_int_to_float """ select int_col from parquet_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_bigint_to_float """ select bigint_col from parquet_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_float_to_float """ select float_col from parquet_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_double_to_float """ select double_col from parquet_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_string_to_float """ select string_col from parquet_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char1_to_float """ select char1_col from parquet_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char2_to_float """ select char2_col from parquet_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_varchar_to_float """ select varchar_col from parquet_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_date_to_float """ select date_col from parquet_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_timestamp_to_float """ select timestamp_col from parquet_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal1_to_float """ select decimal1_col from parquet_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal2_to_float """ select decimal2_col from parquet_primitive_types_to_float order by id """ + } catch (Exception e) { + } + + + try { + qt_parquet_boolean_to_double """ select bool_col from parquet_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_tinyint_to_double """ select tinyint_col from parquet_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_smallint_to_double """ select smallint_col from parquet_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_int_to_double """ select int_col from parquet_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_bigint_to_double """ select bigint_col from parquet_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_float_to_double """ select float_col from parquet_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_double_to_double """ select double_col from parquet_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_string_to_double """ select string_col from parquet_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char1_to_double """ select char1_col from parquet_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char2_to_double """ select char2_col from parquet_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_varchar_to_double """ select varchar_col from parquet_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_date_to_double """ select date_col from parquet_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_timestamp_to_double """ select timestamp_col from parquet_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal1_to_double """ select decimal1_col from parquet_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal2_to_double """ select decimal2_col from parquet_primitive_types_to_double order by id """ + } catch (Exception e) { + } + + + try { + qt_parquet_boolean_to_string """ select bool_col from parquet_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_tinyint_to_string """ select tinyint_col from parquet_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_smallint_to_string """ select smallint_col from parquet_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_int_to_string """ select int_col from parquet_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_bigint_to_string """ select bigint_col from parquet_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_float_to_string """ select float_col from parquet_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_double_to_string """ select double_col from parquet_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_string_to_string """ select string_col from parquet_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char1_to_string """ select char1_col from parquet_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char2_to_string """ select char2_col from parquet_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_varchar_to_string """ select varchar_col from parquet_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_date_to_string """ select date_col from parquet_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_timestamp_to_string """ select timestamp_col from parquet_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal1_to_string """ select decimal1_col from parquet_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal2_to_string """ select decimal2_col from parquet_primitive_types_to_string order by id """ + } catch (Exception e) { + } + + + try { + qt_parquet_boolean_to_date """ select bool_col from parquet_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_tinyint_to_date """ select tinyint_col from parquet_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_smallint_to_date """ select smallint_col from parquet_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_int_to_date """ select int_col from parquet_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_bigint_to_date """ select bigint_col from parquet_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_float_to_date """ select float_col from parquet_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_double_to_date """ select double_col from parquet_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_string_to_date """ select string_col from parquet_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char1_to_date """ select char1_col from parquet_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char2_to_date """ select char2_col from parquet_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_varchar_to_date """ select varchar_col from parquet_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_date_to_date """ select date_col from parquet_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_timestamp_to_date """ select timestamp_col from parquet_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal1_to_date """ select decimal1_col from parquet_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal2_to_date """ select decimal2_col from parquet_primitive_types_to_date order by id """ + } catch (Exception e) { + } + + + + try { + qt_parquet_boolean_to_timestamp """ select bool_col from parquet_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_tinyint_to_timestamp """ select tinyint_col from parquet_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_smallint_to_timestamp """ select smallint_col from parquet_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_int_to_timestamp """ select int_col from parquet_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_bigint_to_timestamp """ select bigint_col from parquet_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_float_to_timestamp """ select float_col from parquet_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_double_to_timestamp """ select double_col from parquet_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_string_to_timestamp """ select string_col from parquet_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char1_to_timestamp """ select char1_col from parquet_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char2_to_timestamp """ select char2_col from parquet_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_varchar_to_timestamp """ select varchar_col from parquet_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_date_to_timestamp """ select date_col from parquet_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_timestamp_to_timestamp """ select timestamp_col from parquet_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal1_to_timestamp """ select decimal1_col from parquet_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal2_to_timestamp """ select decimal2_col from parquet_primitive_types_to_timestamp order by id """ + } catch (Exception e) { + } + + + try { + qt_parquet_boolean_to_decimal1 """ select bool_col from parquet_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_tinyint_to_decimal1 """ select tinyint_col from parquet_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_smallint_to_decimal1 """ select smallint_col from parquet_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_int_to_decimal1 """ select int_col from parquet_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_bigint_to_decimal1 """ select bigint_col from parquet_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_float_to_decimal1 """ select float_col from parquet_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_double_to_decimal1 """ select double_col from parquet_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_string_to_decimal1 """ select string_col from parquet_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char1_to_decimal1 """ select char1_col from parquet_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char2_to_decimal1 """ select char2_col from parquet_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_varchar_to_decimal1 """ select varchar_col from parquet_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_date_to_decimal1 """ select date_col from parquet_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_timestamp_to_decimal1 """ select timestamp_col from parquet_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal1_to_decimal1 """ select decimal1_col from parquet_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal2_to_decimal1 """ select decimal2_col from parquet_primitive_types_to_decimal1 order by id """ + } catch (Exception e) { + } + + + + try { + qt_parquet_boolean_to_decimal2 """ select bool_col from parquet_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_tinyint_to_decimal2 """ select tinyint_col from parquet_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_smallint_to_decimal2 """ select smallint_col from parquet_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_int_to_decimal2 """ select int_col from parquet_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_bigint_to_decimal2 """ select bigint_col from parquet_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_float_to_decimal2 """ select float_col from parquet_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_double_to_decimal2 """ select double_col from parquet_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_string_to_decimal2 """ select string_col from parquet_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char1_to_decimal2 """ select char1_col from parquet_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_char2_to_decimal2 """ select char2_col from parquet_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_varchar_to_decimal2 """ select varchar_col from parquet_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_date_to_decimal2 """ select date_col from parquet_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_timestamp_to_decimal2 """ select timestamp_col from parquet_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal1_to_decimal2 """ select decimal1_col from parquet_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + try { + qt_parquet_decimal2_to_decimal2 """ select decimal2_col from parquet_primitive_types_to_decimal2 order by id """ + } catch (Exception e) { + } + + + sql """ drop catalog ${catalog_name} """ + } + } +}