From 4569d877dad938992834c9eaf73e39a14b5844a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=B3=BD=E7=BF=B0?= <627617031@qq.com> Date: Tue, 6 Feb 2024 13:56:18 +0800 Subject: [PATCH 1/8] Override AbstractJdbcDriver method getTable The default method takes longer to filter after obtaining listTables --- .../main/java/org/dinky/data/model/Table.java | 28 ++++++-- .../metadata/driver/AbstractJdbcDriver.java | 72 ++++++++++--------- 2 files changed, 58 insertions(+), 42 deletions(-) diff --git a/dinky-common/src/main/java/org/dinky/data/model/Table.java b/dinky-common/src/main/java/org/dinky/data/model/Table.java index 1aba079b15..051f61a81a 100644 --- a/dinky-common/src/main/java/org/dinky/data/model/Table.java +++ b/dinky-common/src/main/java/org/dinky/data/model/Table.java @@ -19,12 +19,15 @@ package org.dinky.data.model; +import lombok.Getter; +import lombok.Setter; import org.dinky.assertion.Asserts; import org.dinky.data.enums.TableType; import org.dinky.utils.SqlUtil; import java.beans.Transient; import java.io.Serializable; +import java.sql.SQLException; import java.text.MessageFormat; import java.util.Date; import java.util.HashMap; @@ -32,9 +35,6 @@ import java.util.Map; import java.util.stream.Collectors; -import lombok.Getter; -import lombok.Setter; - /** * Table * @@ -56,14 +56,26 @@ public class Table implements Serializable, Comparable, Cloneable { private Long rows; private Date createTime; private Date updateTime; - /** 表类型 */ + /** + * 表类型 + */ private TableType tableType = TableType.SINGLE_DATABASE_AND_TABLE; - /** 分库或分表对应的表名 */ + /** + * 分库或分表对应的表名 + */ private List schemaTableNameList; private List columns; - public Table() {} + public Table() { + } + + public Table(List columns, String databaseName, String tableName, String tableComment) throws SQLException { + this.name = tableName; + this.schema = databaseName; + this.comment = tableComment; + this.columns = columns; + } public Table(String name, String schema, List columns) { this.name = name; @@ -144,12 +156,14 @@ public String getFlinkTableSql(String catalogName, String flinkConfig) { return String.format("DROP TABLE IF EXISTS %s;\n%s", name, createSql); } + @Override public Object clone() { Table table = null; try { table = (Table) super.clone(); - } catch (CloneNotSupportedException e) { + } catch ( + CloneNotSupportedException e) { e.printStackTrace(); } return table; diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java index c8f6e62ca3..608f43b1d9 100644 --- a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java +++ b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java @@ -19,10 +19,12 @@ package org.dinky.metadata.driver; -import static org.dinky.utils.SplitUtil.contains; -import static org.dinky.utils.SplitUtil.getReValue; -import static org.dinky.utils.SplitUtil.isSplit; - +import cn.hutool.core.text.CharSequenceUtil; +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.pool.DruidPooledConnection; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import lombok.extern.slf4j.Slf4j; import org.dinky.assertion.Asserts; import org.dinky.data.constant.CommonConstant; import org.dinky.data.enums.TableType; @@ -40,35 +42,14 @@ import org.dinky.utils.LogUtil; import org.dinky.utils.TextUtil; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.alibaba.druid.pool.DruidDataSource; -import com.alibaba.druid.pool.DruidPooledConnection; -import com.alibaba.druid.sql.SQLUtils; -import com.alibaba.druid.sql.ast.SQLStatement; - -import cn.hutool.core.text.CharSequenceUtil; -import lombok.extern.slf4j.Slf4j; +import static org.dinky.utils.SplitUtil.*; /** * AbstractJdbcDriver @@ -145,7 +126,9 @@ public Driver connect() { Class.forName(getDriverClass()); DruidPooledConnection connection = createDataSource().getConnection(); conn.set(connection); - } catch (ClassNotFoundException | SQLException e) { + } catch ( + ClassNotFoundException | + SQLException e) { throw new RuntimeException(e); } } @@ -292,6 +275,22 @@ public List
listTables(String schemaName) { return tableList; } + @Override + public Table getTable(String schemaName, String tableName) { + try { + DatabaseMetaData metaData = conn.get().getMetaData(); + ResultSet tables = metaData.getTables(schemaName, null, tableName, new String[]{"TABLE"}); + while (tables.next()) { + String tableComment = tables.getString("REMARKS"); + List columns = listColumns(schemaName, tableName); + return new Table(columns, schemaName, tableName, tableComment); + } + } catch (SQLException e) { + throw new RuntimeException(e); + } + return null; + } + @Override public List listColumns(String schemaName, String tableName) { List columns = new ArrayList<>(); @@ -516,8 +515,11 @@ public int executeUpdate(String sql) throws Exception { } /** - * 标准sql where与order语法都是相同的 不同数据库limit语句不一样，需要单独交由driver去处理，例如oracle 通过{@query(String sql, - * Integer limit)}去截断返回数据，但是在大量数据情况下会导致数据库负载过高。 + * 标准sql where与order语法都是相同的 + * 不同数据库limit语句不一样，需要单独交由driver去处理，例如oracle + * 通过{@query(String sql, + * Integer + * limit)}去截断返回数据，但是在大量数据情况下会导致数据库负载过高。 */ @Override public StringBuilder genQueryOption(QueryData queryData) { @@ -614,7 +616,6 @@ public JdbcSelectResult query(String sql, Integer limit) { /** * 如果执行多条语句返回最后一条语句执行结果 - * */ @Override public JdbcSelectResult executeSql(String sql, Integer limit) { @@ -822,7 +823,8 @@ && contains(tableName, x.get(dbQuery.tableName()))) tableInfo.setUpdateTime( SimpleDateFormat.getDateInstance().parse(updateTime)); } - } catch (ParseException ignored) { + } catch ( + ParseException ignored) { log.warn("set date fail"); } TableType tableType = TableType.type( @@ -836,8 +838,8 @@ && contains(tableName, x.get(dbQuery.tableName()))) + getReValue(x.get(dbQuery.tableName()), splitConfig); List schemaTableNameList = mapList.stream() .filter(y -> (getReValue(y.get(dbQuery.schemaName()), splitConfig) - + "." - + getReValue(y.get(dbQuery.tableName()), splitConfig)) + + "." + + getReValue(y.get(dbQuery.tableName()), splitConfig)) .equals(currentSchemaName)) .map(y -> y.get(dbQuery.schemaName()) + "." + y.get(dbQuery.tableName())) .collect(Collectors.toList()); From 6982f84199b4cc75015717a91d0f07b5d528d21e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=B3=BD=E7=BF=B0?= <627617031@qq.com> Date: Tue, 6 Feb 2024 13:58:22 +0800 Subject: [PATCH 2/8] Override AbstractJdbcDriver method getTable The default method takes longer to filter after obtaining listTables --- .../main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java index 608f43b1d9..3a70bfb85e 100644 --- a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java +++ b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java @@ -286,6 +286,7 @@ public Table getTable(String schemaName, String tableName) { return new Table(columns, schemaName, tableName, tableComment); } } catch (SQLException e) { + log.error("GetTable error:", e); throw new RuntimeException(e); } return null; From 0df416dba7cf9216aa758d2ec00d3a25a8510c95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=B3=BD=E7=BF=B0?= <627617031@qq.com> Date: Tue, 6 Feb 2024 14:33:52 +0800 Subject: [PATCH 3/8] 1 --- .../src/main/java/org/dinky/data/model/Table.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/dinky-common/src/main/java/org/dinky/data/model/Table.java b/dinky-common/src/main/java/org/dinky/data/model/Table.java index 051f61a81a..ee47b21189 100644 --- a/dinky-common/src/main/java/org/dinky/data/model/Table.java +++ b/dinky-common/src/main/java/org/dinky/data/model/Table.java @@ -19,8 +19,6 @@ package org.dinky.data.model; -import lombok.Getter; -import lombok.Setter; import org.dinky.assertion.Asserts; import org.dinky.data.enums.TableType; import org.dinky.utils.SqlUtil; @@ -35,6 +33,9 @@ import java.util.Map; import java.util.stream.Collectors; +import lombok.Getter; +import lombok.Setter; + /** * Table * @@ -67,8 +68,7 @@ public class Table implements Serializable, Comparable
, Cloneable { private List columns; - public Table() { - } + public Table() {} public Table(List columns, String databaseName, String tableName, String tableComment) throws SQLException { this.name = tableName; @@ -156,14 +156,12 @@ public String getFlinkTableSql(String catalogName, String flinkConfig) { return String.format("DROP TABLE IF EXISTS %s;\n%s", name, createSql); } - @Override public Object clone() { Table table = null; try { table = (Table) super.clone(); - } catch ( - CloneNotSupportedException e) { + } catch (CloneNotSupportedException e) { e.printStackTrace(); } return table; From df20badba1303c499fdc7a53812f88f033aec8ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=B3=BD=E7=BF=B0?= <627617031@qq.com> Date: Tue, 6 Feb 2024 15:01:45 +0800 Subject: [PATCH 4/8] mvn spotless:apply --- .../java/org/dinky/metadata/driver/AbstractJdbcDriver.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java index 3a70bfb85e..aa3c31f112 100644 --- a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java +++ b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java @@ -51,6 +51,7 @@ import static org.dinky.utils.SplitUtil.*; + /** * AbstractJdbcDriver * @@ -60,9 +61,8 @@ public abstract class AbstractJdbcDriver extends AbstractDriver { protected ThreadLocal conn = new ThreadLocal<>(); - - private DruidDataSource dataSource; protected String validationQuery = "select 1"; + private DruidDataSource dataSource; abstract String getDriverClass(); From 06206f837c38a91dcd67de40b38e9b77f553257c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=B3=BD=E7=BF=B0?= <627617031@qq.com> Date: Tue, 6 Feb 2024 15:35:06 +0800 Subject: [PATCH 5/8] mvn spotless:apply --- .../metadata/driver/AbstractJdbcDriver.java | 48 ++++++++++++------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java index aa3c31f112..b9dbec4355 100644 --- a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java +++ b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java @@ -19,12 +19,8 @@ package org.dinky.metadata.driver; -import cn.hutool.core.text.CharSequenceUtil; -import com.alibaba.druid.pool.DruidDataSource; -import com.alibaba.druid.pool.DruidPooledConnection; -import com.alibaba.druid.sql.SQLUtils; -import com.alibaba.druid.sql.ast.SQLStatement; -import lombok.extern.slf4j.Slf4j; +import static org.dinky.utils.SplitUtil.*; + import org.dinky.assertion.Asserts; import org.dinky.data.constant.CommonConstant; import org.dinky.data.enums.TableType; @@ -42,15 +38,36 @@ import org.dinky.utils.LogUtil; import org.dinky.utils.TextUtil; -import java.sql.*; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.dinky.utils.SplitUtil.*; +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.pool.DruidPooledConnection; +import com.alibaba.druid.sql.SQLUtils; +import com.alibaba.druid.sql.ast.SQLStatement; +import cn.hutool.core.text.CharSequenceUtil; +import lombok.extern.slf4j.Slf4j; /** * AbstractJdbcDriver @@ -126,9 +143,7 @@ public Driver connect() { Class.forName(getDriverClass()); DruidPooledConnection connection = createDataSource().getConnection(); conn.set(connection); - } catch ( - ClassNotFoundException | - SQLException e) { + } catch (ClassNotFoundException | SQLException e) { throw new RuntimeException(e); } } @@ -279,7 +294,7 @@ public List
listTables(String schemaName) { public Table getTable(String schemaName, String tableName) { try { DatabaseMetaData metaData = conn.get().getMetaData(); - ResultSet tables = metaData.getTables(schemaName, null, tableName, new String[]{"TABLE"}); + ResultSet tables = metaData.getTables(schemaName, null, tableName, new String[] {"TABLE"}); while (tables.next()) { String tableComment = tables.getString("REMARKS"); List columns = listColumns(schemaName, tableName); @@ -824,8 +839,7 @@ && contains(tableName, x.get(dbQuery.tableName()))) tableInfo.setUpdateTime( SimpleDateFormat.getDateInstance().parse(updateTime)); } - } catch ( - ParseException ignored) { + } catch (ParseException ignored) { log.warn("set date fail"); } TableType tableType = TableType.type( @@ -839,8 +853,8 @@ && contains(tableName, x.get(dbQuery.tableName()))) + getReValue(x.get(dbQuery.tableName()), splitConfig); List schemaTableNameList = mapList.stream() .filter(y -> (getReValue(y.get(dbQuery.schemaName()), splitConfig) - + "." - + getReValue(y.get(dbQuery.tableName()), splitConfig)) + + "." + + getReValue(y.get(dbQuery.tableName()), splitConfig)) .equals(currentSchemaName)) .map(y -> y.get(dbQuery.schemaName()) + "." + y.get(dbQuery.tableName())) .collect(Collectors.toList()); From f963ac94d178d1a021b7483b0a0ae656d462faff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=B3=BD=E7=BF=B0?= <627617031@qq.com> Date: Tue, 6 Feb 2024 18:00:11 +0800 Subject: [PATCH 6/8] Add DDL conversion for doris --- .../org/dinky/data/model/SqlGeneration.java | 6 + .../service/impl/DataBaseServiceImpl.java | 8 +- .../org/dinky/data/model/Doris/DorisType.java | 47 ++++ .../data/model/Doris/DorisTypeConverters.java | 24 ++ .../model/Doris/MysqlDorisTypeConverters.java | 208 ++++++++++++++++++ .../main/java/org/dinky/data/model/Table.java | 109 ++++++++- .../metadata/driver/AbstractJdbcDriver.java | 9 +- 7 files changed, 402 insertions(+), 9 deletions(-) create mode 100644 dinky-common/src/main/java/org/dinky/data/model/Doris/DorisType.java create mode 100644 dinky-common/src/main/java/org/dinky/data/model/Doris/DorisTypeConverters.java create mode 100644 dinky-common/src/main/java/org/dinky/data/model/Doris/MysqlDorisTypeConverters.java diff --git a/dinky-admin/src/main/java/org/dinky/data/model/SqlGeneration.java b/dinky-admin/src/main/java/org/dinky/data/model/SqlGeneration.java index ce638a0f84..a1c484033c 100644 --- a/dinky-admin/src/main/java/org/dinky/data/model/SqlGeneration.java +++ b/dinky-admin/src/main/java/org/dinky/data/model/SqlGeneration.java @@ -42,6 +42,12 @@ public class SqlGeneration { notes = "Flink SQL statement for creating a table or view") private String flinkSqlCreate; + @ApiModelProperty( + value = "Doris SQL Create Statement", + dataType = "String", + notes = "Doris SQL statement for creating a table") + private String dorisSqlCreate; + @ApiModelProperty(value = "SQL Select Statement", dataType = "String", notes = "SQL SELECT statement") private String sqlSelect; diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/DataBaseServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/impl/DataBaseServiceImpl.java index 0faf45b9ec..9a9b941598 100644 --- a/dinky-admin/src/main/java/org/dinky/service/impl/DataBaseServiceImpl.java +++ b/dinky-admin/src/main/java/org/dinky/service/impl/DataBaseServiceImpl.java @@ -28,13 +28,6 @@ import org.dinky.data.enums.ProcessStepType; import org.dinky.data.enums.Status; import org.dinky.data.exception.BusException; -import org.dinky.data.model.Column; -import org.dinky.data.model.DataBase; -import org.dinky.data.model.QueryData; -import org.dinky.data.model.Schema; -import org.dinky.data.model.SqlGeneration; -import org.dinky.data.model.Table; -import org.dinky.data.model.Task; import org.dinky.data.result.SqlExplainResult; import org.dinky.job.Job; import org.dinky.job.JobResult; @@ -231,6 +224,7 @@ public SqlGeneration getSqlGeneration(Integer id, String schemaName, String tabl Table table = driver.getTable(schemaName, tableName); SqlGeneration sqlGeneration = new SqlGeneration(); sqlGeneration.setFlinkSqlCreate(table.getFlinkTableSql(dataBase.getName(), dataBase.getFlinkTemplate())); + sqlGeneration.setDorisSqlCreate(table.getDorisTableDDL()); sqlGeneration.setSqlSelect(driver.getSqlSelect(table)); sqlGeneration.setSqlCreate(driver.getCreateTableSql(table)); driver.close(); diff --git a/dinky-common/src/main/java/org/dinky/data/model/Doris/DorisType.java b/dinky-common/src/main/java/org/dinky/data/model/Doris/DorisType.java new file mode 100644 index 0000000000..d0567759fa --- /dev/null +++ b/dinky-common/src/main/java/org/dinky/data/model/Doris/DorisType.java @@ -0,0 +1,47 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dinky.data.model.Doris; + +public class DorisType { + public static final String BOOLEAN = "BOOLEAN"; + public static final String TINYINT = "TINYINT"; + public static final String SMALLINT = "SMALLINT"; + public static final String INT = "INT"; + public static final String BIGINT = "BIGINT"; + public static final String LARGEINT = "LARGEINT"; + public static final String FLOAT = "FLOAT"; + public static final String DOUBLE = "DOUBLE"; + public static final String DECIMAL = "DECIMAL"; + public static final String DECIMAL_V3 = "DECIMALV3"; + public static final String DATE = "DATE"; + public static final String DATE_V2 = "DATEV2"; + public static final String DATETIME = "DATETIME"; + public static final String DATETIME_V2 = "DATETIMEV2"; + public static final String CHAR = "CHAR"; + public static final String VARCHAR = "VARCHAR"; + public static final String STRING = "STRING"; + public static final String HLL = "HLL"; + public static final String BITMAP = "BITMAP"; + public static final String ARRAY = "ARRAY"; + public static final String JSONB = "JSONB"; + public static final String JSON = "JSON"; + public static final String MAP = "MAP"; + public static final String STRUCT = "STRUCT"; +} diff --git a/dinky-common/src/main/java/org/dinky/data/model/Doris/DorisTypeConverters.java b/dinky-common/src/main/java/org/dinky/data/model/Doris/DorisTypeConverters.java new file mode 100644 index 0000000000..2cdbeb715c --- /dev/null +++ b/dinky-common/src/main/java/org/dinky/data/model/Doris/DorisTypeConverters.java @@ -0,0 +1,24 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dinky.data.model.Doris; + +public interface DorisTypeConverters { + String toDorisType(String type, Integer length, Integer scale); +} diff --git a/dinky-common/src/main/java/org/dinky/data/model/Doris/MysqlDorisTypeConverters.java b/dinky-common/src/main/java/org/dinky/data/model/Doris/MysqlDorisTypeConverters.java new file mode 100644 index 0000000000..413e7b5f8e --- /dev/null +++ b/dinky-common/src/main/java/org/dinky/data/model/Doris/MysqlDorisTypeConverters.java @@ -0,0 +1,208 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dinky.data.model.Doris; + +import javax.annotation.Nullable; + +public class MysqlDorisTypeConverters implements DorisTypeConverters { + // MySQL driver returns width of timestamp types instead of precision. + // 19 characters are used for zero-precision timestamps while others + // require 19 + precision + 1 characters with the additional character + // required for the decimal separator. + public static final int MAX_PRECISION = 9; + public static final int MAX_SUPPORTED_DATE_TIME_PRECISION = 6; + private static final int ZERO_PRECISION_TIMESTAMP_COLUMN_SIZE = 19; + private static final String BIT = "BIT"; + private static final String BOOLEAN = "BOOLEAN"; + private static final String BOOL = "BOOL"; + private static final String TINYINT = "TINYINT"; + private static final String TINYINT_UNSIGNED = "TINYINT UNSIGNED"; + private static final String TINYINT_UNSIGNED_ZEROFILL = "TINYINT UNSIGNED ZEROFILL"; + private static final String SMALLINT = "SMALLINT"; + private static final String SMALLINT_UNSIGNED = "SMALLINT UNSIGNED"; + private static final String SMALLINT_UNSIGNED_ZEROFILL = "SMALLINT UNSIGNED ZEROFILL"; + private static final String MEDIUMINT = "MEDIUMINT"; + private static final String MEDIUMINT_UNSIGNED = "MEDIUMINT UNSIGNED"; + private static final String MEDIUMINT_UNSIGNED_ZEROFILL = "MEDIUMINT UNSIGNED ZEROFILL"; + private static final String INT = "INT"; + private static final String INT_UNSIGNED = "INT UNSIGNED"; + private static final String INT_UNSIGNED_ZEROFILL = "INT UNSIGNED ZEROFILL"; + private static final String BIGINT = "BIGINT"; + private static final String SERIAL = "SERIAL"; + private static final String BIGINT_UNSIGNED = "BIGINT UNSIGNED"; + private static final String BIGINT_UNSIGNED_ZEROFILL = "BIGINT UNSIGNED ZEROFILL"; + private static final String REAL = "REAL"; + private static final String REAL_UNSIGNED = "REAL UNSIGNED"; + private static final String REAL_UNSIGNED_ZEROFILL = "REAL UNSIGNED ZEROFILL"; + private static final String FLOAT = "FLOAT"; + private static final String FLOAT_UNSIGNED = "FLOAT UNSIGNED"; + private static final String FLOAT_UNSIGNED_ZEROFILL = "FLOAT UNSIGNED ZEROFILL"; + private static final String DOUBLE = "DOUBLE"; + private static final String DOUBLE_UNSIGNED = "DOUBLE UNSIGNED"; + private static final String DOUBLE_UNSIGNED_ZEROFILL = "DOUBLE UNSIGNED ZEROFILL"; + private static final String DOUBLE_PRECISION = "DOUBLE PRECISION"; + private static final String DOUBLE_PRECISION_UNSIGNED = "DOUBLE PRECISION UNSIGNED"; + private static final String DOUBLE_PRECISION_UNSIGNED_ZEROFILL = "DOUBLE PRECISION UNSIGNED ZEROFILL"; + private static final String NUMERIC = "NUMERIC"; + private static final String NUMERIC_UNSIGNED = "NUMERIC UNSIGNED"; + private static final String NUMERIC_UNSIGNED_ZEROFILL = "NUMERIC UNSIGNED ZEROFILL"; + private static final String FIXED = "FIXED"; + private static final String FIXED_UNSIGNED = "FIXED UNSIGNED"; + private static final String FIXED_UNSIGNED_ZEROFILL = "FIXED UNSIGNED ZEROFILL"; + private static final String DECIMAL = "DECIMAL"; + private static final String DECIMAL_UNSIGNED = "DECIMAL UNSIGNED"; + private static final String DECIMAL_UNSIGNED_ZEROFILL = "DECIMAL UNSIGNED ZEROFILL"; + private static final String CHAR = "CHAR"; + private static final String VARCHAR = "VARCHAR"; + private static final String TINYTEXT = "TINYTEXT"; + private static final String MEDIUMTEXT = "MEDIUMTEXT"; + private static final String TEXT = "TEXT"; + private static final String LONGTEXT = "LONGTEXT"; + private static final String DATE = "DATE"; + private static final String TIME = "TIME"; + private static final String DATETIME = "DATETIME"; + private static final String TIMESTAMP = "TIMESTAMP"; + private static final String YEAR = "YEAR"; + private static final String BINARY = "BINARY"; + private static final String VARBINARY = "VARBINARY"; + private static final String TINYBLOB = "TINYBLOB"; + private static final String MEDIUMBLOB = "MEDIUMBLOB"; + private static final String BLOB = "BLOB"; + private static final String LONGBLOB = "LONGBLOB"; + private static final String JSON = "JSON"; + private static final String ENUM = "ENUM"; + private static final String SET = "SET"; + + private static final String type = "Mysql"; + + public String toDorisType(String type, Integer length, Integer scale) { + switch (type.toUpperCase()) { + case BIT: + case BOOLEAN: + case BOOL: + return DorisType.BOOLEAN; + case TINYINT: + return DorisType.TINYINT; + case TINYINT_UNSIGNED: + case TINYINT_UNSIGNED_ZEROFILL: + case SMALLINT: + return DorisType.SMALLINT; + case SMALLINT_UNSIGNED: + case SMALLINT_UNSIGNED_ZEROFILL: + case INT: + case MEDIUMINT: + case YEAR: + return DorisType.INT; + case INT_UNSIGNED: + case INT_UNSIGNED_ZEROFILL: + case MEDIUMINT_UNSIGNED: + case MEDIUMINT_UNSIGNED_ZEROFILL: + case BIGINT: + return DorisType.BIGINT; + case BIGINT_UNSIGNED: + case BIGINT_UNSIGNED_ZEROFILL: + return DorisType.LARGEINT; + case FLOAT: + case FLOAT_UNSIGNED: + case FLOAT_UNSIGNED_ZEROFILL: + return DorisType.FLOAT; + case REAL: + case REAL_UNSIGNED: + case REAL_UNSIGNED_ZEROFILL: + case DOUBLE: + case DOUBLE_UNSIGNED: + case DOUBLE_UNSIGNED_ZEROFILL: + case DOUBLE_PRECISION: + case DOUBLE_PRECISION_UNSIGNED: + case DOUBLE_PRECISION_UNSIGNED_ZEROFILL: + return DorisType.DOUBLE; + case NUMERIC: + case NUMERIC_UNSIGNED: + case NUMERIC_UNSIGNED_ZEROFILL: + case FIXED: + case FIXED_UNSIGNED: + case FIXED_UNSIGNED_ZEROFILL: + case DECIMAL: + case DECIMAL_UNSIGNED: + case DECIMAL_UNSIGNED_ZEROFILL: + return length != null && length <= 38 + ? String.format( + "%s(%s,%s)", DorisType.DECIMAL_V3, length, scale != null && scale >= 0 ? scale : 0) + : DorisType.STRING; + case DATE: + return DorisType.DATE_V2; + case DATETIME: + case TIMESTAMP: + // default precision is 0 + // see https://dev.mysql.com/doc/refman/8.0/en/date-and-time-type-syntax.html + if (length == null || length <= 0 || length == ZERO_PRECISION_TIMESTAMP_COLUMN_SIZE) { + return String.format("%s(%s)", DorisType.DATETIME_V2, 0); + } else if (length > ZERO_PRECISION_TIMESTAMP_COLUMN_SIZE + 1) { + // Timestamp with a fraction of seconds. + // For example, 2024-01-01 01:01:01.1 + // The decimal point will occupy 1 character. + // Thus,the length of the timestamp is 21. + return String.format( + "%s(%s)", + DorisType.DATETIME_V2, + Math.min( + length - ZERO_PRECISION_TIMESTAMP_COLUMN_SIZE - 1, + MAX_SUPPORTED_DATE_TIME_PRECISION)); + } else if (length <= MAX_PRECISION) { + // For Debezium JSON data, the timestamp/datetime length ranges from 0 to 9. + return String.format( + "%s(%s)", DorisType.DATETIME_V2, Math.min(length, MAX_SUPPORTED_DATE_TIME_PRECISION)); + } else { + throw new UnsupportedOperationException( + "Unsupported length: " + length + " for MySQL TIMESTAMP/DATETIME types"); + } + case CHAR: + case VARCHAR: + checkNotNull(length); + return length * 3 > 65533 ? DorisType.STRING : String.format("%s(%s)", DorisType.VARCHAR, length * 3); + case TINYTEXT: + case TEXT: + case MEDIUMTEXT: + case LONGTEXT: + case ENUM: + case TIME: + case TINYBLOB: + case BLOB: + case MEDIUMBLOB: + case LONGBLOB: + case BINARY: + case VARBINARY: + case SET: + return DorisType.STRING; + case JSON: + return DorisType.JSONB; + default: + throw new UnsupportedOperationException("Unsupported MySQL Type: " + type); + } + } + + public static T checkNotNull(@Nullable T reference) { + if (reference == null) { + throw new NullPointerException(); + } else { + return reference; + } + } +} diff --git a/dinky-common/src/main/java/org/dinky/data/model/Table.java b/dinky-common/src/main/java/org/dinky/data/model/Table.java index ee47b21189..7c807f6523 100644 --- a/dinky-common/src/main/java/org/dinky/data/model/Table.java +++ b/dinky-common/src/main/java/org/dinky/data/model/Table.java @@ -21,12 +21,17 @@ import org.dinky.assertion.Asserts; import org.dinky.data.enums.TableType; +import org.dinky.data.model.Doris.DorisType; +import org.dinky.data.model.Doris.DorisTypeConverters; +import org.dinky.data.model.Doris.MysqlDorisTypeConverters; import org.dinky.utils.SqlUtil; import java.beans.Transient; import java.io.Serializable; import java.sql.SQLException; import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -57,6 +62,7 @@ public class Table implements Serializable, Comparable
, Cloneable { private Long rows; private Date createTime; private Date updateTime; + private List primaryKeys; /** * 表类型 */ @@ -68,13 +74,36 @@ public class Table implements Serializable, Comparable
, Cloneable { private List columns; + private Map columnMap; + + private DorisTypeConverters converters; + public Table() {} - public Table(List columns, String databaseName, String tableName, String tableComment) throws SQLException { + public Table( + List columns, + String databaseName, + String tableName, + String tableComment, + ArrayList primaryKeys, + String DriverType) + throws SQLException { this.name = tableName; this.schema = databaseName; this.comment = tableComment; this.columns = columns; + this.primaryKeys = primaryKeys; + this.columnMap = new HashMap<>(); + switch (DriverType) { + case "MySql": + this.converters = new MysqlDorisTypeConverters(); + } + this.columns.forEach(item -> { + if (this.converters != null) { + item.setType(converters.toDorisType(item.getType(), item.getLength(), item.getScale())); + } + this.columnMap.put(item.getName(), item); + }); } public Table(String name, String schema, List columns) { @@ -156,6 +185,84 @@ public String getFlinkTableSql(String catalogName, String flinkConfig) { return String.format("DROP TABLE IF EXISTS %s;\n%s", name, createSql); } + public static String identifier(String name) { + return "`" + name + "`"; + } + + private static List identifier(List name) { + List result = name.stream().map(m -> identifier(m)).collect(Collectors.toList()); + return result; + } + + @Transient + public String getDorisTableDDL() { + StringBuilder sb = new StringBuilder("CREATE TABLE IF NOT EXISTS "); + sb.append(identifier(schema)).append(".").append(identifier(name)).append("("); + + // append keys + for (String key : primaryKeys) { + buildColumn(sb, columnMap.get(key), true); + } + // append values + for (String key : columnMap.keySet()) { + if (primaryKeys.contains(key)) { + continue; + } + buildColumn(sb, columnMap.get(key), false); + } + sb = sb.deleteCharAt(sb.length() - 1); + sb.append(" ) "); + // append table comment + sb.append(" COMMENT '").append(quoteComment(comment)).append("' "); + // append distribute key + sb.append(" DISTRIBUTED BY HASH(") + .append(String.join(",", identifier(buildDistributeKeys()))) + .append(")"); + sb.append(" BUCKETS AUTO "); + return sb.toString(); + } + + private static void buildColumn(StringBuilder sql, Column field, boolean isKey) { + String fieldType = field.getType(); + if (isKey && DorisType.STRING.equals(fieldType)) { + fieldType = String.format("%s(%s)", DorisType.VARCHAR, 65533); + } + sql.append(identifier(field.getName())).append(" ").append(fieldType); + + if (field.getDefaultValue() != null) { + sql.append(" DEFAULT " + quoteDefaultValue(field.getDefaultValue())); + } + sql.append(" COMMENT '").append(quoteComment(field.getComment())).append("',"); + } + + public static String quoteDefaultValue(String defaultValue) { + // DEFAULT current_timestamp not need quote + if (defaultValue.equalsIgnoreCase("current_timestamp")) { + return defaultValue; + } + return "'" + defaultValue + "'"; + } + + public static String quoteComment(String comment) { + if (comment == null) { + return ""; + } else { + return comment.replaceAll("'", "\\\\'"); + } + } + + private List buildDistributeKeys() { + if (!this.primaryKeys.isEmpty()) { + return primaryKeys; + } + if (!this.columnMap.isEmpty()) { + Map.Entry firstField = + this.columnMap.entrySet().iterator().next(); + return Collections.singletonList(firstField.getKey()); + } + return new ArrayList<>(); + } + @Override public Object clone() { Table table = null; diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java index b9dbec4355..3ec7a1cda4 100644 --- a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java +++ b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java @@ -298,7 +298,14 @@ public Table getTable(String schemaName, String tableName) { while (tables.next()) { String tableComment = tables.getString("REMARKS"); List columns = listColumns(schemaName, tableName); - return new Table(columns, schemaName, tableName, tableComment); + ArrayList primaryKeys = new ArrayList<>(); + ResultSet rs = metaData.getPrimaryKeys(schemaName, null, tableName); + while (rs.next()) { + String fieldName = rs.getString("COLUMN_NAME"); + primaryKeys.add(fieldName); + } + + return new Table(columns, schemaName, tableName, tableComment, primaryKeys, getType()); } } catch (SQLException e) { log.error("GetTable error:", e); From b6be20fd29d8188959f5c46ea84cbaa286ff07c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=B3=BD=E7=BF=B0?= <627617031@qq.com> Date: Tue, 6 Feb 2024 21:11:07 +0800 Subject: [PATCH 7/8] Fix code style --- .../java/org/dinky/service/impl/DataBaseServiceImpl.java | 7 +++++++ .../java/org/dinky/metadata/driver/AbstractJdbcDriver.java | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/DataBaseServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/impl/DataBaseServiceImpl.java index 9a9b941598..b1e7fab867 100644 --- a/dinky-admin/src/main/java/org/dinky/service/impl/DataBaseServiceImpl.java +++ b/dinky-admin/src/main/java/org/dinky/service/impl/DataBaseServiceImpl.java @@ -28,6 +28,13 @@ import org.dinky.data.enums.ProcessStepType; import org.dinky.data.enums.Status; import org.dinky.data.exception.BusException; +import org.dinky.data.model.Column; +import org.dinky.data.model.DataBase; +import org.dinky.data.model.QueryData; +import org.dinky.data.model.Schema; +import org.dinky.data.model.SqlGeneration; +import org.dinky.data.model.Table; +import org.dinky.data.model.Task; import org.dinky.data.result.SqlExplainResult; import org.dinky.job.Job; import org.dinky.job.JobResult; diff --git a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java index 3ec7a1cda4..d5cca3f33a 100644 --- a/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java +++ b/dinky-metadata/dinky-metadata-base/src/main/java/org/dinky/metadata/driver/AbstractJdbcDriver.java @@ -19,7 +19,9 @@ package org.dinky.metadata.driver; -import static org.dinky.utils.SplitUtil.*; +import static org.dinky.utils.SplitUtil.contains; +import static org.dinky.utils.SplitUtil.getReValue; +import static org.dinky.utils.SplitUtil.isSplit; import org.dinky.assertion.Asserts; import org.dinky.data.constant.CommonConstant; From 2bcf10994b2a2be02760f358253c3a3cc908bca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=B3=BD=E7=BF=B0?= <627617031@qq.com> Date: Wed, 7 Feb 2024 10:33:22 +0800 Subject: [PATCH 8/8] Add a default 2doristype converter --- .../Doris/DefaultDorisTypeConverters.java | 99 +++++++++++++++++++ .../main/java/org/dinky/data/model/Table.java | 11 ++- 2 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 dinky-common/src/main/java/org/dinky/data/model/Doris/DefaultDorisTypeConverters.java diff --git a/dinky-common/src/main/java/org/dinky/data/model/Doris/DefaultDorisTypeConverters.java b/dinky-common/src/main/java/org/dinky/data/model/Doris/DefaultDorisTypeConverters.java new file mode 100644 index 0000000000..76ff4b7bea --- /dev/null +++ b/dinky-common/src/main/java/org/dinky/data/model/Doris/DefaultDorisTypeConverters.java @@ -0,0 +1,99 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.dinky.data.model.Doris; + +public class DefaultDorisTypeConverters implements DorisTypeConverters { + public static final String STRING = "java.lang.String"; + public static final String INT = "int"; + public static final String LONG = "long"; + public static final String DOUBLE = "double"; + public static final String BOOLEAN = "java.lang.Boolean"; + public static final String JAVA_LANG_BOOLEAN = "Boolean"; + public static final String DATE = "java.sql.Date"; + public static final String TIMESTAMP = "java.sql.Timestamp"; + public static final String DECIMAL = "java.math.BigDecimal"; + public static final String BYTE = "java.lang.Byte"; + public static final String JAVA_LANG_BYTE = "Byte"; + public static final String JAVA_LANG_SHORT = "java.lang.Short"; + public static final String SHORT = "short"; + public static final String INTEGER = "java.lang.Integer"; + public static final String JAVA_LANG_LONG = "java.lang.Long"; + public static final String JAVA_LANG_FLOAT = "java.lang.Float"; + public static final String FLOAT = "float"; + public static final String JAVA_LANG_DOUBLE = "java.lang.Double"; + public static final String LOCAL_DATE = "java.time.LocalDate"; + public static final String LOCAL_TIME = "java.time.LocalTime"; + public static final String TIME = "java.sql.Time"; + public static final String LOCAL_DATETIME = "java.time.LocalDateTime"; + public static final String OFFSET_DATETIME = "java.time.OffsetDateTime"; + public static final String INSTANT = "java.time.Instant"; + public static final String DURATION = "java.time.Duration"; + public static final String PERIOD = "java.time.Period"; + public static final String BYTES = "byte[]"; + public static final String T = "T[]"; + public static final String MAP = "java.util.Map"; + + @Override + public String toDorisType(String type, Integer length, Integer scale) { + switch (type) { + case SHORT: + case JAVA_LANG_SHORT: + return DorisType.SMALLINT; + case INTEGER: + case LONG: + case JAVA_LANG_LONG: + case INT: + return DorisType.INT; + case FLOAT: + case JAVA_LANG_FLOAT: + return DorisType.FLOAT; + case DOUBLE: + case JAVA_LANG_DOUBLE: + return DorisType.DOUBLE; + case BOOLEAN: + case JAVA_LANG_BOOLEAN: + return DorisType.BOOLEAN; + case STRING: + case TIME: + case DECIMAL: + case TIMESTAMP: + case LOCAL_TIME: + case LOCAL_DATETIME: + case OFFSET_DATETIME: + case INSTANT: + case DURATION: + case PERIOD: + case BYTES: + return DorisType.STRING; + case DATE: + case LOCAL_DATE: + return DorisType.DATE; + case BYTE: + case JAVA_LANG_BYTE: + return DorisType.TINYINT; + case T: + return DorisType.ARRAY; + case MAP: + return DorisType.MAP; + default: + throw new UnsupportedOperationException("Unsupported type: " + type); + } + } +} diff --git a/dinky-common/src/main/java/org/dinky/data/model/Table.java b/dinky-common/src/main/java/org/dinky/data/model/Table.java index 7c807f6523..88234d87b3 100644 --- a/dinky-common/src/main/java/org/dinky/data/model/Table.java +++ b/dinky-common/src/main/java/org/dinky/data/model/Table.java @@ -21,6 +21,7 @@ import org.dinky.assertion.Asserts; import org.dinky.data.enums.TableType; +import org.dinky.data.model.Doris.DefaultDorisTypeConverters; import org.dinky.data.model.Doris.DorisType; import org.dinky.data.model.Doris.DorisTypeConverters; import org.dinky.data.model.Doris.MysqlDorisTypeConverters; @@ -95,11 +96,17 @@ public Table( this.primaryKeys = primaryKeys; this.columnMap = new HashMap<>(); switch (DriverType) { - case "MySql": + case "Mysql": this.converters = new MysqlDorisTypeConverters(); + break; + default: + this.converters = new DefaultDorisTypeConverters(); } this.columns.forEach(item -> { - if (this.converters != null) { + if (this.converters instanceof DefaultDorisTypeConverters) { + item.setType( + converters.toDorisType(item.getJavaType().getJavaType(), item.getLength(), item.getScale())); + } else { item.setType(converters.toDorisType(item.getType(), item.getLength(), item.getScale())); } this.columnMap.put(item.getName(), item);