diff --git a/example/session/src/main/java/org/apache/iotdb/TableModelSessionPoolExample.java b/example/session/src/main/java/org/apache/iotdb/TableModelSessionPoolExample.java index 0a335f02c969..cbf404b96765 100644 --- a/example/session/src/main/java/org/apache/iotdb/TableModelSessionPoolExample.java +++ b/example/session/src/main/java/org/apache/iotdb/TableModelSessionPoolExample.java @@ -113,7 +113,7 @@ public static void main(String[] args) { int rowIndex = tablet.getRowSize(); tablet.addTimestamp(rowIndex, timestamp); tablet.addValue("region_id", rowIndex, "1"); - tablet.addValue("plant_id", rowIndex, "5"); + tablet.addValue("plant_id", rowIndex, null); tablet.addValue("device_id", rowIndex, "3"); tablet.addValue("model", rowIndex, "A"); tablet.addValue("temperature", rowIndex, 37.6F); diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java index 6cb7e0082426..e7f4b228d4f1 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppCommonConfig.java @@ -651,6 +651,12 @@ public CommonConfig setAuditableOperationResult(String auditableOperationResult) return this; } + @Override + public CommonConfig setRestrictObjectLimit(boolean restrictObjectLimit) { + setProperty("restrict_object_limit", String.valueOf(restrictObjectLimit)); + return this; + } + // For part of the log directory public String getClusterConfigStr() { return fromConsensusFullNameToAbbr(properties.getProperty(CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS)) diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java index 1b4801f56922..abb0f8bf8bb0 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/config/MppSharedCommonConfig.java @@ -684,4 +684,11 @@ public CommonConfig setAuditableOperationResult(String auditableOperationResult) cnConfig.setAuditableOperationResult(auditableOperationResult); return this; } + + @Override + public CommonConfig setRestrictObjectLimit(boolean restrictObjectLimit) { + cnConfig.setRestrictObjectLimit(restrictObjectLimit); + dnConfig.setRestrictObjectLimit(restrictObjectLimit); + return this; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java index f08c085bc6c4..96a0fbe27e05 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java @@ -169,6 +169,10 @@ public String getSystemDir() { return getDataNodeDir() + File.separator + "system"; } + public String getDataNodeObjectDir() { + return getDataNodeDir() + File.separator + "data" + File.separator + "object"; + } + @Override protected MppJVMConfig initVMConfig() { return MppJVMConfig.builder() diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java index fdcff20dbc85..148046423cd0 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteCommonConfig.java @@ -477,4 +477,9 @@ public CommonConfig setAuditableOperationLevel(String auditableOperationLevel) { public CommonConfig setAuditableOperationResult(String auditableOperationResult) { return this; } + + @Override + public CommonConfig setRestrictObjectLimit(boolean restrictObjectLimit) { + return this; + } } diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java index f27bdc8d66da..531f94eec4b1 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java @@ -211,4 +211,6 @@ default CommonConfig setDefaultDatabaseLevel(int defaultDatabaseLevel) { CommonConfig setAuditableOperationLevel(String auditableOperationLevel); CommonConfig setAuditableOperationResult(String auditableOperationResult); + + CommonConfig setRestrictObjectLimit(boolean restrictObjectLimit); } diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestResultSet.java b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestResultSet.java index e98ff7e7c144..64c18db11a8c 100644 --- a/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestResultSet.java +++ b/integration-test/src/main/java/org/apache/iotdb/itbase/runtime/ClusterTestResultSet.java @@ -781,8 +781,12 @@ public Ref getRef(int columnIndex) { } @Override - public Blob getBlob(int columnIndex) { - throw new UnsupportedOperationException(); + public Blob getBlob(int columnIndex) throws SQLException { + RequestDelegate delegate = createLocalRequestDelegate(); + for (ResultSet rs : resultSets) { + delegate.addRequest(() -> rs.getBlob(columnIndex)); + } + return delegate.requestAllAndCompare(); } @Override @@ -806,8 +810,12 @@ public Ref getRef(String columnLabel) { } @Override - public Blob getBlob(String columnLabel) { - throw new UnsupportedOperationException(); + public Blob getBlob(String columnLabel) throws SQLException { + RequestDelegate delegate = createLocalRequestDelegate(); + for (ResultSet rs : resultSets) { + delegate.addRequest(() -> rs.getBlob(columnLabel)); + } + return delegate.requestAllAndCompare(); } @Override diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQueryIT.java new file mode 100644 index 000000000000..65c0f09e4138 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQueryIT.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.object; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.RowRecord; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.jdbc.IoTDBJDBCResultSet.OBJECT_ERR_MSG; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBObjectQueryIT { + + private static final String DATABASE_NAME = "test_db"; + + private static final String TIME_ZONE = "+00:00"; + + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE t1(device_id STRING TAG, o1 OBJECT, b1 BLOB, s1 STRING)", + "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(1, 'd1', X'cafebabe01', to_object(true, 0, X'cafebabe01'), 'cafebabe01')", + "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(2, 'd1', X'cafebabe0202', to_object(true, 0, X'cafebabe02'), 'cafebabe02')", + "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(3, 'd1', X'cafebabe0303', to_object(true, 0, X'cafebabe03'), 'cafebabe03')", + "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(4, 'd1', X'cafebabe04', to_object(true, 0, X'cafebabe04'), 'cafebabe04')", + "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(1, 'd2', X'cafebade01', to_object(true, 0, X'cafebade01'), 'cafebade01')", + "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(2, 'd2', X'cafebade0202', to_object(true, 0, X'cafebade02'), 'cafebade02')", + "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(3, 'd2', X'cafebade0302', to_object(true, 0, X'cafebade03'), 'cafebade03')", + "INSERT INTO t1(time, device_id, b1, o1, s1) VALUES(4, 'd2', X'cafebade04', to_object(true, 0, X'cafebade04'), 'cafebade04')", + "FLUSH", + }; + + @BeforeClass + public static void classSetUp() { + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void classTearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void jdbcTest() { + try (Connection connection = + EnvFactory.getEnv() + .getConnection( + SessionConfig.DEFAULT_USER, + SessionConfig.DEFAULT_PASSWORD, + BaseEnv.TABLE_SQL_DIALECT)) { + connection.setClientInfo("time_zone", TIME_ZONE); + try (Statement statement = connection.createStatement()) { + statement.execute("use " + DATABASE_NAME); + try (ResultSet resultSet = + statement.executeQuery( + "SELECT time, b1, o1, s1 FROM t1 WHERE device_id = 'd1' ORDER BY time")) { + int cnt = 0; + while (resultSet.next()) { + cnt++; + try { + resultSet.getBlob(3); + fail(); + } catch (SQLException e) { + assertEquals(OBJECT_ERR_MSG, e.getMessage()); + } + + try { + resultSet.getBytes("o1"); + fail(); + } catch (SQLException e) { + assertEquals(OBJECT_ERR_MSG, e.getMessage()); + } + + String s = resultSet.getString(3); + assertEquals("(Object) 5 B", s); + } + assertEquals(4, cnt); + } + + try (ResultSet resultSet = + statement.executeQuery( + "SELECT time, b1, READ_OBJECT(o1), s1 FROM t1 WHERE device_id = 'd2' AND READ_OBJECT(o1)=b1 ORDER BY time")) { + int cnt = 0; + String[] ans = {"0xcafebade01", "0xcafebade04"}; + while (resultSet.next()) { + String s = resultSet.getString(3); + assertEquals(ans[cnt], s); + cnt++; + } + assertEquals(2, cnt); + } + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void sessionTest() { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE " + DATABASE_NAME); + + // SessionDataSet + try (SessionDataSet dataSet = + session.executeQueryStatement( + "SELECT time, b1, o1, s1 FROM t1 WHERE device_id = 'd1' ORDER BY time")) { + int cnt = 0; + while (dataSet.hasNext()) { + cnt++; + RowRecord rowRecord = dataSet.next(); + Field field = rowRecord.getField(2); + String s = field.getStringValue(); + assertEquals("(Object) 5 B", s); + Object blob = field.getObjectValue(TSDataType.OBJECT); + assertTrue(blob instanceof String); + assertEquals("(Object) 5 B", blob); + + try { + field.getBinaryV(); + fail(); + } catch (UnsupportedOperationException e) { + assertEquals("OBJECT Type only support getStringValue", e.getMessage()); + } + } + assertEquals(4, cnt); + } + + // SessionDataSet.DataIterator + try (SessionDataSet dataSet = + session.executeQueryStatement( + "SELECT time, b1, o1, s1 FROM t1 WHERE device_id = 'd2' ORDER BY time")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + int cnt = 0; + while (iterator.next()) { + cnt++; + Object o = iterator.getObject(3); + assertTrue(o instanceof String); + assertEquals("(Object) 5 B", o); + String s = iterator.getString("o1"); + assertEquals("(Object) 5 B", s); + try { + iterator.getBlob(3); + fail(); + } catch (StatementExecutionException e) { + assertEquals("OBJECT Type only support getString", e.getMessage()); + } + } + assertEquals(4, cnt); + } + } catch (IoTDBConnectionException | StatementExecutionException e) { + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQueryIT2.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQueryIT2.java new file mode 100644 index 000000000000..9a0dd60907e4 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/object/IoTDBObjectQueryIT2.java @@ -0,0 +1,296 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.object; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.utils.Binary; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.Statement; +import java.time.LocalDate; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBObjectQueryIT2 { + + private static final String DATABASE_NAME = "test"; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setDataReplicationFactor(1); + EnvFactory.getEnv().initClusterEnvironment(); + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE " + DATABASE_NAME); + statement.execute("USE " + DATABASE_NAME); + statement.execute( + "CREATE TABLE table1(device STRING TAG, s4 DATE FIELD, s5 TIMESTAMP FIELD, s6 BLOB FIELD, s7 STRING FIELD, s8 OBJECT FIELD, s9 OBJECT FIELD)"); + for (int i = 1; i <= 10; i++) { + for (int j = 0; j < 10; j++) { + statement.execute( + String.format( + "insert into table1(time, device, s4, s5, s6, s7, s8) " + + "values(%d, '%s', '%s', %d, %s, '%s', %s)", + j, + "d" + i, + LocalDate.of(2024, 5, i % 31 + 1), + j, + "X'cafebabe'", + j, + "to_object(true, 0, X'cafebabe')")); + } + } + } + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testObjectLength() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE " + DATABASE_NAME); + SessionDataSet sessionDataSet = + session.executeQueryStatement("select length(s8) from table1 limit 1"); + SessionDataSet.DataIterator iterator = sessionDataSet.iterator(); + while (iterator.next()) { + long length = iterator.getLong(1); + Assert.assertEquals(4, length); + } + } + } + + @Test + public void testReadObject() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE " + DATABASE_NAME); + SessionDataSet sessionDataSet = + session.executeQueryStatement("select read_object(s8) from table1 where device = 'd2'"); + SessionDataSet.DataIterator iterator = sessionDataSet.iterator(); + byte[] expected = new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE}; + while (iterator.next()) { + Binary blob = iterator.getBlob(1); + Assert.assertArrayEquals(expected, blob.getValues()); + } + + sessionDataSet = + session.executeQueryStatement( + "select read_object(s8, 1) from table1 where device = 'd3'"); + iterator = sessionDataSet.iterator(); + expected = new byte[] {(byte) 0xFE, (byte) 0xBA, (byte) 0xBE}; + while (iterator.next()) { + Binary blob = iterator.getBlob(1); + Assert.assertArrayEquals(expected, blob.getValues()); + } + + sessionDataSet = + session.executeQueryStatement( + "select read_object(s8, 1, 2) from table1 where device = 'd1'"); + iterator = sessionDataSet.iterator(); + expected = new byte[] {(byte) 0xFE, (byte) 0xBA}; + while (iterator.next()) { + Binary blob = iterator.getBlob(1); + Assert.assertArrayEquals(expected, blob.getValues()); + } + + sessionDataSet = + session.executeQueryStatement( + "select read_object(s8, 1, 1000) from table1 where device = 'd1'"); + iterator = sessionDataSet.iterator(); + expected = new byte[] {(byte) 0xFE, (byte) 0xBA, (byte) 0xBE}; + while (iterator.next()) { + Binary blob = iterator.getBlob(1); + Assert.assertArrayEquals(expected, blob.getValues()); + } + + sessionDataSet = + session.executeQueryStatement( + "select count(*) from table1 where device = 'd1' and s6 = read_object(s8)"); + iterator = sessionDataSet.iterator(); + while (iterator.next()) { + long count = iterator.getLong(1); + Assert.assertEquals(10, count); + } + + // read_object are not pushed down. Read remote files + sessionDataSet = + session.executeQueryStatement( + "select read_object(t1_s8) from (select t1.s8 as t1_s8, t2.s8 as t2_s8 from table1 as t1 inner join table1 as t2 using(time))"); + iterator = sessionDataSet.iterator(); + expected = new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE}; + while (iterator.next()) { + Binary blob = iterator.getBlob(1); + Assert.assertArrayEquals(expected, blob.getValues()); + } + } + } + + @Test + public void testFunctionAndClauses() + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE " + DATABASE_NAME); + + SessionDataSet sessionDataSet = + session.executeQueryStatement( + "select length(s8) from table1 where device = 'd2' and s8 is not null limit 1"); + SessionDataSet.DataIterator iterator = sessionDataSet.iterator(); + while (iterator.next()) { + Assert.assertEquals(4, iterator.getLong(1)); + } + sessionDataSet = + session.executeQueryStatement( + "select count(s8), first(s8), last(s8), first_by(s8, time), last_by(s8, time) from table1 where device = 'd1' and cast(s8 as string) = '(Object) 4 B' and try_cast(s8 as string) = '(Object) 4 B'"); + iterator = sessionDataSet.iterator(); + while (iterator.next()) { + Assert.assertEquals(10, iterator.getLong(1)); + Assert.assertEquals("(Object) 4 B", iterator.getString(2)); + Assert.assertEquals("(Object) 4 B", iterator.getString(3)); + Assert.assertEquals("(Object) 4 B", iterator.getString(4)); + Assert.assertEquals("(Object) 4 B", iterator.getString(5)); + } + + sessionDataSet = session.executeQueryStatement("select coalesce(s9, s8) from table1"); + iterator = sessionDataSet.iterator(); + while (iterator.next()) { + Assert.assertEquals("(Object) 4 B", iterator.getString(1)); + } + + // MATCH_RECOGNIZE + Assert.assertThrows( + StatementExecutionException.class, + () -> + session.executeNonQueryStatement( + "select m.cnt from table1 match_recognize (order by s8 measures RPR_LAST(time) as cnt one row per match pattern (B+) define B as B.s6 = prev(B.s6)) as m")); + Assert.assertThrows( + StatementExecutionException.class, + () -> + session.executeNonQueryStatement( + "select m.cnt from table1 match_recognize (partition by s8 measures RPR_LAST(time) as cnt one row per match pattern (B+) define B as B.s6 = prev(B.s6)) as m")); + + sessionDataSet = + session.executeQueryStatement( + "select m.value from table1 match_recognize(partition by s6 measures prev(s8) as value one row per match pattern (B+) define B as B.s6=prev(B.s6)) as m"); + iterator = sessionDataSet.iterator(); + while (iterator.next()) { + Assert.assertEquals("(Object) 4 B", iterator.getString(1)); + } + + // WHERE + session.executeQueryStatement( + "select time, s8 from table1 where device = 'd10' and s8 is not null"); + iterator = sessionDataSet.iterator(); + while (iterator.next()) { + Assert.assertEquals("(Object) 4 B", iterator.getString(2)); + } + + // GROUP BY + Assert.assertThrows( + StatementExecutionException.class, + () -> session.executeNonQueryStatement("select count(*) from table1 group by s8")); + + // ORDER BY + Assert.assertThrows( + StatementExecutionException.class, + () -> session.executeNonQueryStatement("select count(*) from table1 order by s8")); + + // FILL + Assert.assertThrows( + StatementExecutionException.class, + () -> + session.executeNonQueryStatement( + "select time, s8 from table1 where device = 'd10' fill method linear")); + session.executeQueryStatement( + "select time, s8 from table1 where device = 'd10' fill method previous"); + iterator = sessionDataSet.iterator(); + while (iterator.next()) { + Assert.assertEquals("(Object) 4 B", iterator.getString(2)); + } + + // HAVING + session.executeQueryStatement( + "select device, count(s8) from table1 group by device having count(s8) > 0"); + iterator = sessionDataSet.iterator(); + while (iterator.next()) { + long count = iterator.getLong(2); + Assert.assertEquals(10, count); + } + + // WINDOW + Assert.assertThrows( + StatementExecutionException.class, + () -> + session.executeNonQueryStatement( + "select *, nth_value(s8,2) over(partition by s8) from table1")); + Assert.assertThrows( + StatementExecutionException.class, + () -> + session.executeNonQueryStatement( + "select *, nth_value(s8,2) over(order by s8) from table1")); + session.executeNonQueryStatement( + "select *, nth_value(s8,2) over(partition by device) from table1"); + session.executeNonQueryStatement( + "select *, lead(s8) over(partition by device order by time) from table1"); + session.executeNonQueryStatement( + "select *, first_value(s8) over(partition by device) from table1"); + session.executeNonQueryStatement( + "select *, last_value(s8) over(partition by device) from table1"); + session.executeNonQueryStatement( + "select *, lag(s8) over(partition by device order by time) from table1"); + + // Table-value function + Assert.assertThrows( + StatementExecutionException.class, + () -> + session.executeNonQueryStatement( + "select * from session(data => table1 partition by s8, timecol => 'time', gap => 1ms)")); + Assert.assertThrows( + StatementExecutionException.class, + () -> + session.executeNonQueryStatement( + "select * from session(data => table1 order by s8, timecol => 'time', gap => 1ms)")); + sessionDataSet = + session.executeQueryStatement( + "select * from hop(data => table1, timecol => 'time', slide => 1ms, size => 2ms)"); + iterator = sessionDataSet.iterator(); + while (iterator.next()) { + String str = iterator.getString("s8"); + Assert.assertEquals("(Object) 4 B", str); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBSimpleQueryTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBSimpleQueryTableIT.java index 57a187950e27..b97dc4915a1d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBSimpleQueryTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/IoTDBSimpleQueryTableIT.java @@ -657,19 +657,26 @@ public void testNewDataType() { statement.execute("CREATE DATABASE test"); statement.execute("USE " + DATABASE_NAME); statement.execute( - "CREATE TABLE table1(device STRING TAG, s4 DATE FIELD, s5 TIMESTAMP FIELD, s6 BLOB FIELD, s7 STRING FIELD)"); + "CREATE TABLE table1(device STRING TAG, " + + "s4 DATE FIELD, s5 TIMESTAMP FIELD, s6 BLOB FIELD, s7 STRING FIELD, s8 OBJECT FIELD)"); for (int i = 1; i <= 10; i++) { statement.execute( String.format( - "insert into table1(time, device, s4, s5, s6, s7) values(%d, 'd1', '%s', %d, %s, '%s')", - i, LocalDate.of(2024, 5, i % 31 + 1), i, "X'cafebabe'", i)); + "insert into table1(time, device, s4, s5, s6, s7, s8) " + + "values(%d, 'd1', '%s', %d, %s, '%s', %s)", + i, + LocalDate.of(2024, 5, i % 31 + 1), + i, + "X'cafebabe'", + i, + "to_object(true, 0, X'cafebabe')")); } try (ResultSet resultSet = statement.executeQuery("select * from table1")) { final ResultSetMetaData metaData = resultSet.getMetaData(); final int columnCount = metaData.getColumnCount(); - assertEquals(6, columnCount); + assertEquals(7, columnCount); HashMap columnType = new HashMap<>(); for (int i = 3; i <= columnCount; i++) { if (metaData.getColumnLabel(i).equals("s4")) { @@ -680,6 +687,8 @@ public void testNewDataType() { columnType.put(i, TSDataType.BLOB); } else if (metaData.getColumnLabel(i).equals("s7")) { columnType.put(i, TSDataType.TEXT); + } else if (metaData.getColumnLabel(i).equals("s8")) { + columnType.put(i, TSDataType.OBJECT); } } byte[] byteArray = new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE}; @@ -689,12 +698,58 @@ public void testNewDataType() { long timestamp = resultSet.getLong(4); byte[] blob = resultSet.getBytes(5); String text = resultSet.getString(6); + String objectSizeString = resultSet.getString(7); assertEquals(2024 - 1900, date.getYear()); assertEquals(5 - 1, date.getMonth()); assertEquals(time % 31 + 1, date.getDate()); assertEquals(time, timestamp); assertArrayEquals(byteArray, blob); assertEquals(String.valueOf(time), text); + assertEquals("(Object) 4 B", objectSizeString); + } + } + try (ResultSet resultSet = statement.executeQuery("select read_object(s8) from table1")) { + final ResultSetMetaData metaData = resultSet.getMetaData(); + final int columnCount = metaData.getColumnCount(); + assertEquals(1, columnCount); + byte[] byteArray = new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE}; + while (resultSet.next()) { + byte[] blob = resultSet.getBytes(1); + assertArrayEquals(byteArray, blob); + } + } + + } catch (SQLException e) { + fail(); + } + } + + @Test + public void testObjectDataType() { + try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE test"); + statement.execute("USE " + DATABASE_NAME); + statement.execute("CREATE TABLE table1(device STRING TAG, s8 OBJECT FIELD)"); + statement.execute( + "insert into table1(time, device, s8) values(1, 'd1', to_object(false, 0, X'cafe'))"); + statement.execute( + "insert into table1(time, device, s8) values(1, 'd1', to_object(true, 2, X'babe'))"); + + try (ResultSet resultSet = statement.executeQuery("select * from table1")) { + while (resultSet.next()) { + String objectSizeString = resultSet.getString(3); + assertEquals("(Object) 4 B", objectSizeString); + } + } + try (ResultSet resultSet = statement.executeQuery("select read_object(s8) from table1")) { + final ResultSetMetaData metaData = resultSet.getMetaData(); + final int columnCount = metaData.getColumnCount(); + assertEquals(1, columnCount); + byte[] byteArray = new byte[] {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE}; + while (resultSet.next()) { + byte[] blob = resultSet.getBytes(1); + assertArrayEquals(byteArray, blob); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBScalarFunctionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBScalarFunctionTableIT.java index 01c2d591918e..0e4b0a38221c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBScalarFunctionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/old/builtinfunction/scalar/IoTDBScalarFunctionTableIT.java @@ -1370,56 +1370,56 @@ public void lengthTestFail() { tableAssertTestFail( "select s1,Length(s1,1) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", + + ": Scalar function length only accepts one argument and it must be text or string or blob or object data type.", DATABASE_NAME); // case 2: wrong data type tableAssertTestFail( "select s1,Length(s2) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", + + ": Scalar function length only accepts one argument and it must be text or string or blob or object data type.", DATABASE_NAME); // case 3: wrong data type tableAssertTestFail( "select s1,Length(s3) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", + + ": Scalar function length only accepts one argument and it must be text or string or blob or object data type.", DATABASE_NAME); // case 4: wrong data type tableAssertTestFail( "select s1,Length(s4) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", + + ": Scalar function length only accepts one argument and it must be text or string or blob or object data type.", DATABASE_NAME); // case 5: wrong data type tableAssertTestFail( "select s1,Length(s5) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", + + ": Scalar function length only accepts one argument and it must be text or string or blob or object data type.", DATABASE_NAME); // case 6: wrong data type tableAssertTestFail( "select s1,Length(s6) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", + + ": Scalar function length only accepts one argument and it must be text or string or blob or object data type.", DATABASE_NAME); // case 7: wrong data type tableAssertTestFail( "select s1,Length(s7) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", + + ": Scalar function length only accepts one argument and it must be text or string or blob or object data type.", DATABASE_NAME); // case 8: wrong data type tableAssertTestFail( "select s1,Length(s8) from lengthTable", TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text, string, or blob data type.", + + ": Scalar function length only accepts one argument and it must be text or string or blob or object data type.", DATABASE_NAME); } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBLengthFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBLengthFunctionIT.java index 57bf23918d59..7f3ba13b11d5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBLengthFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBLengthFunctionIT.java @@ -107,7 +107,7 @@ public void testLengthOnBlob() { public void testLengthFunctionOnInvalidInputs() { String expectedErrorMessage = TSStatusCode.SEMANTIC_ERROR.getStatusCode() - + ": Scalar function length only accepts one argument and it must be text, string, or blob data type."; + + ": Scalar function length only accepts one argument and it must be text or string or blob or object data type."; // Exception 1: Using LENGTH() on non-TEXT/BLOB/STRING types tableAssertTestFail("SELECT length(c_int) FROM table1", expectedErrorMessage, DATABASE_NAME); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index 753ff5fa42b0..5eaf230b6008 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -41,6 +41,9 @@ import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Paths; import java.sql.Connection; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -777,6 +780,134 @@ public void testConcurrentAutoCreateAndDropColumn() throws Exception { } } + @Test + public void testTableObjectCheck() throws Exception { + final Set illegal = new HashSet<>(Arrays.asList("./", ".", "..", ".\\", "../hack")); + for (final String single : illegal) { + testObject4SingleIllegalPath(single); + } + } + + private void testObject4SingleIllegalPath(final String illegal) throws Exception { + try (final Connection connection = + EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement(); + final ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + statement.execute("create database if not exists db2"); + statement.execute("use db2"); + statement.execute(String.format("create table \"%s\" ()", illegal)); + + try { + statement.execute(String.format("alter table \"%s\" add column a object", illegal)); + fail(); + } catch (final SQLException e) { + Assert.assertEquals( + String.format( + "701: When there are object fields, the tableName %s shall not be '.', '..' or contain './', '.\\'", + illegal), + e.getMessage()); + } + + // Test auto-create + String testObject = + System.getProperty("user.dir") + + File.separator + + "target" + + File.separator + + "test-classes" + + File.separator + + "object-example.pt"; + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("a", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("b", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("c", TSDataType.INT32)); + schemaList.add(new MeasurementSchema(illegal, TSDataType.OBJECT)); + final List columnTypes = + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.ATTRIBUTE, + ColumnCategory.FIELD, + ColumnCategory.FIELD); + final Tablet tablet = + new Tablet( + illegal, + IMeasurementSchema.getMeasurementNameList(schemaList), + IMeasurementSchema.getDataTypeList(schemaList), + columnTypes, + 1); + tablet.addTimestamp(0, System.currentTimeMillis()); + tablet.addValue(schemaList.get(0).getMeasurementName(), 0, "d1"); + tablet.addValue(schemaList.get(1).getMeasurementName(), 0, "a1"); + tablet.addValue(schemaList.get(2).getMeasurementName(), 0, 0); + tablet.addValue(0, 3, true, 0, Files.readAllBytes(Paths.get(testObject))); + + try { + session.executeNonQueryStatement("use db2"); + session.insert(tablet); + } catch (final Exception e) { + Assert.assertEquals( + String.format( + "701: When there are object fields, the tableName %s shall not be '.', '..' or contain './', '.\\'", + illegal), + e.getMessage()); + } + + try { + statement.execute(String.format("create table test (\"%s\" object)", illegal)); + fail(); + } catch (final SQLException e) { + Assert.assertEquals( + String.format( + "701: When there are object fields, the objectName %s shall not be '.', '..' or contain './', '.\\'", + illegal), + e.getMessage()); + } + + statement.execute("create table test (a tag, b attribute, c int32, d object)"); + + // Cannot auto-extend illegal column + tablet.setTableName("test"); + try { + session.executeNonQueryStatement("use db2"); + session.insert(tablet); + } catch (final Exception e) { + Assert.assertEquals( + String.format( + "701: When there are object fields, the objectName %s shall not be '.', '..' or contain './', '.\\'", + illegal), + e.getMessage()); + } + + // It's OK if you don't write object + statement.execute(String.format("insert into test (a, b, c) values ('%s', 1, 1)", illegal)); + try { + statement.execute( + String.format("insert into test (a, b, c, d) values ('%s', 1, 1, 's')", illegal)); + fail(); + } catch (final SQLException e) { + Assert.assertEquals( + String.format( + "507: When there are object fields, the deviceId [test, %s] shall not be '.', '..' or contain './', '.\\'", + illegal), + e.getMessage()); + } + + try { + statement.execute(String.format("alter table test add column \"%s\" object", illegal)); + fail(); + } catch (final SQLException e) { + Assert.assertEquals( + String.format( + "701: When there are object fields, the objectName %s shall not be '.', '..' or contain './', '.\\'", + illegal), + e.getMessage()); + } + + statement.execute("drop database db2"); + } + } + @Test public void testTreeViewTable() throws Exception { try (final Connection connection = EnvFactory.getEnv().getConnection(); diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectInsertIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectInsertIT.java new file mode 100644 index 000000000000..08946bf4cceb --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectInsertIT.java @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.relational.it.session; + +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.rpc.TSStatusCode; + +import com.google.common.io.BaseEncoding; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.write.record.Tablet; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertNull; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBObjectInsertIT { + + @BeforeClass + public static void classSetUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @Before + public void setUp() throws Exception { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS db1"); + } + } + + @After + public void tearDown() throws Exception { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("DROP DATABASE IF EXISTS db1"); + } + } + + @AfterClass + public static void classTearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void insertObjectTest() + throws IoTDBConnectionException, StatementExecutionException, IOException { + String testObject = + System.getProperty("user.dir") + + File.separator + + "target" + + File.separator + + "test-classes" + + File.separator + + "object-example.pt"; + File object = new File(testObject); + + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // insert table data by tablet + List columnNameList = + Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file"); + List dataTypeList = + Arrays.asList( + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.FLOAT, + TSDataType.OBJECT); + List columnTypeList = + new ArrayList<>( + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD)); + Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1); + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, 1); + tablet.addValue(rowIndex, 0, "1"); + tablet.addValue(rowIndex, 1, "5"); + tablet.addValue(rowIndex, 2, "3"); + tablet.addValue(rowIndex, 3, 37.6F); + tablet.addValue(rowIndex, 4, true, 0, Files.readAllBytes(Paths.get(testObject))); + session.insert(tablet); + tablet.reset(); + + try (SessionDataSet dataSet = + session.executeQueryStatement("select file from object_table where time = 1")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + Assert.assertEquals( + BytesUtils.parseObjectByteArrayToString(BytesUtils.longToBytes(object.length())), + iterator.getString(1)); + } + } + + try (SessionDataSet dataSet = + session.executeQueryStatement( + "select READ_OBJECT(file) from object_table where time = 1")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + Binary binary = iterator.getBlob(1); + Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)), binary.getValues()); + } + } + } + // test object file path + boolean success = false; + for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { + String objectDirStr = dataNodeWrapper.getDataNodeObjectDir(); + File objectDir = new File(objectDirStr); + if (objectDir.exists() && objectDir.isDirectory()) { + File[] regionDirs = objectDir.listFiles(); + if (regionDirs != null) { + for (File regionDir : regionDirs) { + if (regionDir.isDirectory()) { + File objectFile = + new File( + regionDir, + convertPathString("object_table") + + File.separator + + convertPathString("1") + + File.separator + + convertPathString("5") + + File.separator + + convertPathString("3") + + File.separator + + convertPathString("file") + + File.separator + + "1.bin"); + if (objectFile.exists() && objectFile.isFile()) { + success = true; + } + } + } + } + } + } + Assert.assertTrue(success); + } + + @Test + public void insertObjectSegmentsTest() + throws IoTDBConnectionException, StatementExecutionException, IOException { + String testObject = + System.getProperty("user.dir") + + File.separator + + "target" + + File.separator + + "test-classes" + + File.separator + + "object-example.pt"; + byte[] objectBytes = Files.readAllBytes(Paths.get(testObject)); + List objectSegments = new ArrayList<>(); + for (int i = 0; i < objectBytes.length; i += 512) { + objectSegments.add(Arrays.copyOfRange(objectBytes, i, Math.min(i + 512, objectBytes.length))); + } + + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // insert table data by tablet + List columnNameList = + Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file"); + List dataTypeList = + Arrays.asList( + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.FLOAT, + TSDataType.OBJECT); + List columnTypeList = + new ArrayList<>( + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD)); + Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1); + for (int i = 0; i < objectSegments.size() - 1; i++) { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, 1); + tablet.addValue(rowIndex, 0, "1"); + tablet.addValue(rowIndex, 1, "5"); + tablet.addValue(rowIndex, 2, "3"); + tablet.addValue(rowIndex, 3, 37.6F); + tablet.addValue(rowIndex, 4, false, i * 512L, objectSegments.get(i)); + session.insert(tablet); + tablet.reset(); + } + + try (SessionDataSet dataSet = + session.executeQueryStatement("select file from object_table where time = 1")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + assertNull(iterator.getString(1)); + } + } + + // insert segment with wrong offset + try { + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, 1); + tablet.addValue(rowIndex, 0, "1"); + tablet.addValue(rowIndex, 1, "5"); + tablet.addValue(rowIndex, 2, "3"); + tablet.addValue(rowIndex, 3, 37.6F); + tablet.addValue(rowIndex, 4, false, 512L, objectSegments.get(1)); + session.insert(tablet); + } catch (StatementExecutionException e) { + Assert.assertEquals(TSStatusCode.OBJECT_INSERT_ERROR.getStatusCode(), e.getStatusCode()); + Assert.assertEquals( + String.format( + "741: The file length %d is not equal to the offset %d", + ((objectSegments.size() - 1) * 512), 512L), + e.getMessage()); + } finally { + tablet.reset(); + } + + // last segment + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, 1); + tablet.addValue(rowIndex, 0, "1"); + tablet.addValue(rowIndex, 1, "5"); + tablet.addValue(rowIndex, 2, "3"); + tablet.addValue(rowIndex, 3, 37.6F); + tablet.addValue( + rowIndex, + 4, + true, + (objectSegments.size() - 1) * 512L, + objectSegments.get(objectSegments.size() - 1)); + session.insert(tablet); + tablet.reset(); + + try (SessionDataSet dataSet = + session.executeQueryStatement( + "select READ_OBJECT(file) from object_table where time = 1")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + Binary binary = iterator.getBlob(1); + Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)), binary.getValues()); + } + } + } + + // test object file path + boolean success = false; + for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { + String objectDirStr = dataNodeWrapper.getDataNodeObjectDir(); + File objectDir = new File(objectDirStr); + if (objectDir.exists() && objectDir.isDirectory()) { + File[] regionDirs = objectDir.listFiles(); + if (regionDirs != null) { + for (File regionDir : regionDirs) { + if (regionDir.isDirectory()) { + File objectFile = + new File( + regionDir, + convertPathString("object_table") + + File.separator + + convertPathString("1") + + File.separator + + convertPathString("5") + + File.separator + + convertPathString("3") + + File.separator + + convertPathString("file") + + File.separator + + "1.bin"); + if (objectFile.exists() && objectFile.isFile()) { + success = true; + } + } + } + } + } + } + Assert.assertTrue(success); + } + + protected String convertPathString(String path) { + return BaseEncoding.base32().omitPadding().encode(path.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectInsertIT2.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectInsertIT2.java new file mode 100644 index 000000000000..9bd16938f563 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBObjectInsertIT2.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.session; + +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.write.record.Tablet; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBObjectInsertIT2 extends IoTDBObjectInsertIT { + + @BeforeClass + public static void classSetUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setRestrictObjectLimit(true); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void classTearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void changeRestrictObjectLimitTest() + throws IoTDBConnectionException, StatementExecutionException, IOException { + EnvFactory.getEnv().getConfig().getCommonConfig().setRestrictObjectLimit(false); + TestUtils.restartCluster(EnvFactory.getEnv()); + String testObject = + System.getProperty("user.dir") + + File.separator + + "target" + + File.separator + + "test-classes" + + File.separator + + "object-example.pt"; + File object = new File(testObject); + + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("USE \"db1\""); + // insert table data by tablet + List columnNameList = + Arrays.asList("region_id", "plant_id", "device_id", "temperature", "file"); + List dataTypeList = + Arrays.asList( + TSDataType.STRING, + TSDataType.STRING, + TSDataType.STRING, + TSDataType.FLOAT, + TSDataType.OBJECT); + List columnTypeList = + new ArrayList<>( + Arrays.asList( + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.TAG, + ColumnCategory.FIELD, + ColumnCategory.FIELD)); + Tablet tablet = new Tablet("object_table", columnNameList, dataTypeList, columnTypeList, 1); + int rowIndex = tablet.getRowSize(); + tablet.addTimestamp(rowIndex, 1); + tablet.addValue(rowIndex, 0, "1"); + tablet.addValue(rowIndex, 1, "5"); + tablet.addValue(rowIndex, 2, "3"); + tablet.addValue(rowIndex, 3, 37.6F); + tablet.addValue(rowIndex, 4, true, 0, Files.readAllBytes(Paths.get(testObject))); + session.insert(tablet); + tablet.reset(); + + try (SessionDataSet dataSet = + session.executeQueryStatement("select file from object_table where time = 1")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + Assert.assertEquals( + BytesUtils.parseObjectByteArrayToString(BytesUtils.longToBytes(object.length())), + iterator.getString(1)); + } + } + + try (SessionDataSet dataSet = + session.executeQueryStatement( + "select READ_OBJECT(file) from object_table where time = 1")) { + SessionDataSet.DataIterator iterator = dataSet.iterator(); + while (iterator.next()) { + Binary binary = iterator.getBlob(1); + Assert.assertArrayEquals(Files.readAllBytes(Paths.get(testObject)), binary.getValues()); + } + } + } + // test object file path + boolean success = false; + for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { + String objectDirStr = dataNodeWrapper.getDataNodeObjectDir(); + File objectDir = new File(objectDirStr); + if (objectDir.exists() && objectDir.isDirectory()) { + File[] regionDirs = objectDir.listFiles(); + if (regionDirs != null) { + for (File regionDir : regionDirs) { + if (regionDir.isDirectory()) { + File objectFile = + new File( + regionDir, + convertPathString("object_table") + + File.separator + + convertPathString("1") + + File.separator + + convertPathString("5") + + File.separator + + convertPathString("3") + + File.separator + + convertPathString("file") + + File.separator + + "1.bin"); + if (objectFile.exists() && objectFile.isFile()) { + success = true; + } + } + } + } + } + } + Assert.assertTrue(success); + } + + @Override + protected String convertPathString(String path) { + return path; + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java index fa24f595a801..9c02ac94208a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java @@ -1627,7 +1627,6 @@ public void insertRelationalTabletWithAutoCastTest() throws IoTDBConnectionException, StatementExecutionException { int testNum = 14; Set dataTypes = TSDataTypeTestUtils.getSupportedTypes(); - try { for (TSDataType from : dataTypes) { for (TSDataType to : dataTypes) { diff --git a/integration-test/src/test/resources/object-example.pt b/integration-test/src/test/resources/object-example.pt new file mode 100644 index 000000000000..67d4aec6999f Binary files /dev/null and b/integration-test/src/test/resources/object-example.pt differ diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java index 9093133631a0..59819ee418e6 100644 --- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/type/Type.java @@ -49,7 +49,10 @@ public enum Type { BLOB((byte) 10), /* STRING */ - STRING((byte) 11); + STRING((byte) 11), + + /* OBJECT */ + OBJECT((byte) 12); private final byte dataType; diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/access/Record.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/access/Record.java index 8c6e2a7f3578..538b8c296c12 100644 --- a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/access/Record.java +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/relational/access/Record.java @@ -23,7 +23,9 @@ import org.apache.tsfile.utils.Binary; +import java.io.File; import java.time.LocalDate; +import java.util.Optional; public interface Record { /** @@ -94,7 +96,7 @@ public interface Record { * Returns the String value at the specified column in this row. * *

Users need to ensure that the data type of the specified column is {@code TSDataType.TEXT} - * or {@code TSDataType.STRING}. + * or {@code TSDataType.STRING} or {@code TSDataType.OBJECT}. * * @param columnIndex index of the specified column * @return the String value at the specified column in this row @@ -113,6 +115,46 @@ public interface Record { Object getObject(int columnIndex); + /** + * Returns the OBJECT value's real file path in current node at the specified column in this row. + * + * @param columnIndex index of the specified column + * @return Optional.empty() if current node doesn't have the real file storing the object content, + * otherwise the File referring to the OBJECT value's real file path + */ + Optional getObjectFile(int columnIndex); + + /** + * Returns the Binary representation of an object stored at the specified column in this row. + * + *

Users need to ensure that the data type of the specified column is {@code + * TSDataType.OBJECT}. + * + *

This method returns the entire binary data of the object and may require considerable memory + * if the stored object is large. + * + * @param columnIndex index of the specified column + * @return the Binary content of the object at the specified column + */ + Binary readObject(int columnIndex); + + /** + * Returns a partial Binary segment of an object stored at the specified column in this row. + * + *

Users need to ensure that the data type of the specified column is {@code + * TSDataType.OBJECT}. + * + *

This method enables reading a subset of the stored object without materializing the entire + * binary data in memory, which is useful for large objects and streaming access patterns. + * + * @param columnIndex index of the specified column + * @param offset byte offset of the subsection read + * @param length number of bytes to read starting from the offset. If length < 0, read the entire + * binary data from offset. + * @return the Binary content of the object segment at the specified column + */ + Binary readObject(int columnIndex, long offset, int length); + /** * Returns the actual data type of the value at the specified column in this row. * diff --git a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java index a5c2852ac1b9..c4c38c285dce 100644 --- a/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java +++ b/iotdb-api/udf-api/src/main/java/org/apache/iotdb/udf/api/type/Type.java @@ -56,7 +56,11 @@ public enum Type { BLOB((byte) 10), /* STRING */ - STRING((byte) 11); + STRING((byte) 11), + + /* OBJECT */ + OBJECT((byte) 12); + private final byte dataType; Type(byte type) { @@ -92,6 +96,7 @@ public boolean checkObjectType(Object o) { case DATE: return o instanceof LocalDate; case BLOB: + case OBJECT: return o instanceof Binary; case STRING: case TEXT: @@ -102,7 +107,8 @@ public boolean checkObjectType(Object o) { } public static List allTypes() { - return Arrays.asList(BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT, TIMESTAMP, DATE, BLOB, STRING); + return Arrays.asList( + BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TEXT, TIMESTAMP, DATE, BLOB, STRING, OBJECT); } public static List numericTypes() { diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java index aee48bd9927d..17587cf8811a 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java @@ -742,6 +742,7 @@ private static String getStringByColumnIndex( case DOUBLE: case TEXT: case STRING: + case OBJECT: return resultSet.getString(columnIndex); case BLOB: byte[] v = resultSet.getBytes(columnIndex); diff --git a/iotdb-client/client-cpp/src/main/Common.cpp b/iotdb-client/client-cpp/src/main/Common.cpp index cac4448b8e1d..56dc96830571 100644 --- a/iotdb-client/client-cpp/src/main/Common.cpp +++ b/iotdb-client/client-cpp/src/main/Common.cpp @@ -89,6 +89,7 @@ TSDataType::TSDataType getDataTypeByStr(const std::string& typeStr) { if (typeStr == "DATE") return TSDataType::DATE; if (typeStr == "BLOB") return TSDataType::BLOB; if (typeStr == "STRING") return TSDataType::STRING; + if (typeStr == "OBJECT") return TSDataType::OBJECT; return TSDataType::UNKNOWN; } diff --git a/iotdb-client/client-cpp/src/main/Common.h b/iotdb-client/client-cpp/src/main/Common.h index 129e458c67b6..aee444986057 100644 --- a/iotdb-client/client-cpp/src/main/Common.h +++ b/iotdb-client/client-cpp/src/main/Common.h @@ -95,7 +95,8 @@ enum TSDataType { TIMESTAMP = (char)8, DATE = (char)9, BLOB = (char)10, - STRING = (char)11 + STRING = (char)11, + OBJECT = (char)12 }; } diff --git a/iotdb-client/client-cpp/src/main/IoTDBRpcDataSet.cpp b/iotdb-client/client-cpp/src/main/IoTDBRpcDataSet.cpp index 7339d74c2c34..38bef0ab7928 100644 --- a/iotdb-client/client-cpp/src/main/IoTDBRpcDataSet.cpp +++ b/iotdb-client/client-cpp/src/main/IoTDBRpcDataSet.cpp @@ -448,6 +448,7 @@ std::string IoTDBRpcDataSet::getStringByTsBlockColumnIndexAndDataType(int32_t in return std::to_string(curTsBlock_->getColumn(index)->getDouble(tsBlockIndex_)); case TSDataType::TEXT: case TSDataType::STRING: + case TSDataType::OBJECT: case TSDataType::BLOB: { auto binary = curTsBlock_->getColumn(index)->getBinary(tsBlockIndex_); return binary->getStringValue(); diff --git a/iotdb-client/client-cpp/src/main/Session.cpp b/iotdb-client/client-cpp/src/main/Session.cpp index 8182a67cccb6..da1263527d1e 100644 --- a/iotdb-client/client-cpp/src/main/Session.cpp +++ b/iotdb-client/client-cpp/src/main/Session.cpp @@ -59,6 +59,8 @@ TSDataType::TSDataType getTSDataTypeFromString(const string& str) { return TSDataType::BLOB; } else if (str == "STRING") { return TSDataType::STRING; + } else if (str == "OBJECT") { + return TSDataType::OBJECT; } return TSDataType::UNKNOWN; } @@ -88,6 +90,7 @@ void Tablet::createColumns() { break; case TSDataType::STRING: case TSDataType::BLOB: + case TSDataType::OBJECT: case TSDataType::TEXT: values[i] = new string[maxRowNumber]; break; @@ -135,6 +138,7 @@ void Tablet::deleteColumns() { } case TSDataType::STRING: case TSDataType::BLOB: + case TSDataType::OBJECT: case TSDataType::TEXT: { string* valueBuf = (string*)(values[i]); delete[] valueBuf; @@ -182,6 +186,7 @@ void Tablet::deepCopyTabletColValue(void* const* srcPtr, void** destPtr, TSDataT } case TSDataType::STRING: case TSDataType::TEXT: + case TSDataType::OBJECT: case TSDataType::BLOB: { *destPtr = new std::string[maxRowNumber]; std::string* srcStr = static_cast(src); @@ -232,6 +237,7 @@ size_t Tablet::getValueByteSize() { break; case TSDataType::STRING: case TSDataType::BLOB: + case TSDataType::OBJECT: case TSDataType::TEXT: { valueOccupation += rowSize * 4; string* valueBuf = (string*)(values[i]); @@ -361,6 +367,7 @@ string SessionUtils::getValue(const Tablet& tablet) { } case TSDataType::STRING: case TSDataType::BLOB: + case TSDataType::OBJECT: case TSDataType::TEXT: { string* valueBuf = (string*)(tablet.values[i]); for (size_t index = 0; index < tablet.rowSize; index++) { @@ -582,6 +589,7 @@ void Session::sortTablet(Tablet& tablet) { } case TSDataType::STRING: case TSDataType::BLOB: + case TSDataType::OBJECT: case TSDataType::TEXT: { sortValuesList((string*)(tablet.values[i]), index, tablet.rowSize); break; @@ -654,6 +662,7 @@ Session::putValuesIntoBuffer(const vector& types, const break; case TSDataType::STRING: case TSDataType::BLOB: + case TSDataType::OBJECT: case TSDataType::TEXT: { int32_t len = (uint32_t)strlen(values[i]); appendValues(buf, (char*)(&len), sizeof(uint32_t)); @@ -689,6 +698,8 @@ int8_t Session::getDataTypeNumber(TSDataType::TSDataType type) { return 10; case TSDataType::STRING: return 11; + case TSDataType::OBJECT: + return 12; default: return -1; } @@ -1352,16 +1363,20 @@ void Session::buildInsertTabletReq(TSInsertTabletReq& request, Tablet& tablet, b sortTablet(tablet); } - request.prefixPath = tablet.deviceId; + request.__set_prefixPath(tablet.deviceId); - request.measurements.reserve(tablet.schemas.size()); - request.types.reserve(tablet.schemas.size()); + std::vector reqMeasurements; + reqMeasurements.reserve(tablet.schemas.size()); + std::vector types; + types.reserve(tablet.schemas.size()); for (pair schema : tablet.schemas) { - request.measurements.push_back(schema.first); - request.types.push_back(schema.second); + reqMeasurements.push_back(schema.first); + types.push_back(schema.second); } - request.values = move(SessionUtils::getValue(tablet)); - request.timestamps = move(SessionUtils::getTime(tablet)); + request.__set_measurements(reqMeasurements); + request.__set_types(types); + request.__set_values(SessionUtils::getValue(tablet)); + request.__set_timestamps(SessionUtils::getTime(tablet)); request.__set_size(tablet.rowSize); request.__set_isAligned(tablet.isAligned); } @@ -1446,6 +1461,7 @@ void Session::insertRelationalTablet(Tablet& tablet, bool sorted) { } case TSDataType::STRING: case TSDataType::TEXT: + case TSDataType::OBJECT: case TSDataType::BLOB: { currentTablet.addValue(tablet.schemas[col].first, rowIndex, *(string*)tablet.getValue(col, row, tablet.schemas[col].second)); diff --git a/iotdb-client/client-cpp/src/main/Session.h b/iotdb-client/client-cpp/src/main/Session.h index 35c05fad22c4..4b901d5b20fe 100644 --- a/iotdb-client/client-cpp/src/main/Session.h +++ b/iotdb-client/client-cpp/src/main/Session.h @@ -322,6 +322,58 @@ class Tablet { } } + // Add a Binary value with extra metadata: [isEOF (1 byte)] + [offset (8 bytes)] + [actual content] + void addValue(size_t schemaId, size_t rowIndex, bool isEOF, int64_t offset, const std::vector& content) { + // Check schemaId bounds + if (schemaId >= schemas.size()) { + char tmpStr[100]; + sprintf(tmpStr, + "Tablet::addBinaryValueWithMeta(), schemaId >= schemas.size(). schemaId=%ld, schemas.size()=%ld.", + schemaId, schemas.size()); + throw std::out_of_range(tmpStr); + } + + // Check rowIndex bounds + if (rowIndex >= rowSize) { + char tmpStr[100]; + sprintf(tmpStr, "Tablet::addBinaryValueWithMeta(), rowIndex >= rowSize. rowIndex=%ld, rowSize=%ld.", + rowIndex, rowSize); + throw std::out_of_range(tmpStr); + } + + TSDataType::TSDataType dataType = schemas[schemaId].second; + if (dataType != TSDataType::OBJECT) { + throw std::invalid_argument("The data type of schemaId " + std::to_string(schemaId) + " is not OBJECT."); + } + + // Create a byte array of size [1 (isEOF) + 8 (offset) + content size] + std::vector val(content.size() + 9); + + // Write the isEOF flag (1 byte) + val[0] = isEOF ? 1 : 0; + + // Write the 8-byte offset in big-endian order + for (int i = 0; i < 8; ++i) { + val[1 + i] = static_cast((offset >> (56 - i * 8)) & 0xFF); + } + + // Append the content bytes + std::copy(content.begin(), content.end(), val.begin() + 9); + + // Cast the value array and assign the Binary data (stored as string) + std::string valEncoded = std::string(reinterpret_cast(val.data()), val.size()); + safe_cast(valEncoded, ((string*)values[schemaId])[rowIndex]); + } + + void addValue(const string& schemaName, size_t rowIndex, bool isEOF, int64_t offset, + const std::vector& content) { + if (schemaNameIndex.find(schemaName) == schemaNameIndex.end()) { + throw SchemaNotFoundException(string("Schema ") + schemaName + " not found."); + } + size_t schemaId = schemaNameIndex[schemaName]; + addValue(schemaId, rowIndex, isEOF, offset, content); + } + template void addValue(const string& schemaName, size_t rowIndex, const T& value) { if (schemaNameIndex.find(schemaName) == schemaNameIndex.end()) { @@ -331,7 +383,6 @@ class Tablet { addValue(schemaId, rowIndex, value); } - void* getValue(size_t schemaId, size_t rowIndex, TSDataType::TSDataType dataType) { if (schemaId >= schemas.size()) { throw std::out_of_range("Tablet::getValue schemaId out of range: " @@ -358,6 +409,7 @@ class Tablet { return &(reinterpret_cast(values[schemaId])[rowIndex]); case TSDataType::BLOB: case TSDataType::STRING: + case TSDataType::OBJECT: case TSDataType::TEXT: return &(reinterpret_cast(values[schemaId])[rowIndex]); default: diff --git a/iotdb-client/client-cpp/src/main/SessionDataSet.cpp b/iotdb-client/client-cpp/src/main/SessionDataSet.cpp index 048d53409161..0cdd13402e4d 100644 --- a/iotdb-client/client-cpp/src/main/SessionDataSet.cpp +++ b/iotdb-client/client-cpp/src/main/SessionDataSet.cpp @@ -98,6 +98,7 @@ std::string RowRecord::toString() { break; case TSDataType::BLOB: case TSDataType::STRING: + case TSDataType::OBJECT: case TSDataType::TEXT: if (!fields[i].stringV.is_initialized()) { ret.append("null"); @@ -268,6 +269,7 @@ shared_ptr SessionDataSet::constructRowRecordFromValueArray() { case TSDataType::TEXT: case TSDataType::BLOB: case TSDataType::STRING: + case TSDataType::OBJECT: field.stringV = iotdbRpcDataSet_->getBinary(columnName)->getStringValue(); break; default: diff --git a/iotdb-client/client-cpp/src/test/cpp/sessionRelationalIT.cpp b/iotdb-client/client-cpp/src/test/cpp/sessionRelationalIT.cpp index 94f956b56ea4..8dff31ba7f4a 100644 --- a/iotdb-client/client-cpp/src/test/cpp/sessionRelationalIT.cpp +++ b/iotdb-client/client-cpp/src/test/cpp/sessionRelationalIT.cpp @@ -159,7 +159,8 @@ TEST_CASE("Test RelationalTabletTsblockRead", "[testRelationalTabletTsblockRead] "field7 TIMESTAMP field," "field8 DATE field," "field9 BLOB field," - "field10 STRING field)"); + "field10 STRING field," + "field11 OBJECT field)"); vector> schemaList; schemaList.push_back(make_pair("field1", TSDataType::BOOLEAN)); @@ -172,8 +173,9 @@ TEST_CASE("Test RelationalTabletTsblockRead", "[testRelationalTabletTsblockRead] schemaList.push_back(make_pair("field8", TSDataType::DATE)); schemaList.push_back(make_pair("field9", TSDataType::BLOB)); schemaList.push_back(make_pair("field10", TSDataType::STRING)); + schemaList.push_back(make_pair("field11", TSDataType::OBJECT)); - vector columnTypes(10, ColumnCategory::FIELD); + vector columnTypes(11, ColumnCategory::FIELD); int64_t timestamp = 0; int maxRowNumber = 50000; @@ -192,6 +194,9 @@ TEST_CASE("Test RelationalTabletTsblockRead", "[testRelationalTabletTsblockRead] tablet.addValue(7, rowIndex, boost::gregorian::date(2025, 5, 15)); tablet.addValue(8, rowIndex, "blob_" + to_string(row)); tablet.addValue(9, rowIndex, "string_" + to_string(row)); + vector rawData = {0x01, 0x02, 0x03, 0x04}; + // always non-null + tablet.addValue(10, rowIndex, true, 0, rawData); if (row % 2 == 0) { for (int col = 0; col <= 9; col++) { @@ -227,6 +232,7 @@ TEST_CASE("Test RelationalTabletTsblockRead", "[testRelationalTabletTsblockRead] REQUIRE_FALSE(dataIter.getDateByIndex(9).is_initialized()); REQUIRE_FALSE(dataIter.getStringByIndex(10).is_initialized()); REQUIRE_FALSE(dataIter.getStringByIndex(11).is_initialized()); + REQUIRE_FALSE(!dataIter.getStringByIndex(12).is_initialized()); } else { REQUIRE(dataIter.getLongByIndex(1).value() == timestamp + rowNum); REQUIRE(dataIter.getBooleanByIndex(2).value() == (rowNum % 2 == 0)); @@ -239,6 +245,8 @@ TEST_CASE("Test RelationalTabletTsblockRead", "[testRelationalTabletTsblockRead] REQUIRE(dataIter.getDateByIndex(9).value() == boost::gregorian::date(2025, 5, 15)); REQUIRE(dataIter.getStringByIndex(10).value() == "blob_" + to_string(rowNum)); REQUIRE(dataIter.getStringByIndex(11).value() == "string_" + to_string(rowNum)); + // [isEOF (1 byte)] + [offset (8 bytes)] + [content (4 bytes)] + REQUIRE(dataIter.getStringByIndex(12).value() != ""); } rowNum++; } diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java index fa67708fed0c..9a1fcd291486 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/SessionDataSet.java @@ -189,6 +189,7 @@ private RowRecord constructRowRecordFromValueArray() throws StatementExecutionEx case TEXT: case BLOB: case STRING: + case OBJECT: field.setBinaryV(ioTDBRpcDataSet.getBinary(columnName)); break; default: @@ -326,10 +327,26 @@ public LocalDate getDate(String columnName) throws StatementExecutionException { } public Binary getBlob(int columnIndex) throws StatementExecutionException { + final TSDataType dataType = ioTDBRpcDataSet.getDataType(columnIndex); + if (dataType == null) { + return null; + } + + if (dataType.equals(TSDataType.OBJECT)) { + throw new StatementExecutionException("OBJECT Type only support getString"); + } return ioTDBRpcDataSet.getBinary(columnIndex); } public Binary getBlob(String columnName) throws StatementExecutionException { + final TSDataType dataType = ioTDBRpcDataSet.getDataType(columnName); + if (dataType == null) { + return null; + } + + if (dataType.equals(TSDataType.OBJECT)) { + throw new StatementExecutionException("OBJECT Type only support getString"); + } return ioTDBRpcDataSet.getBinary(columnName); } diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java index 8baf1f9b6f21..f14e2ee6b327 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java @@ -396,6 +396,7 @@ public static ByteBuffer convertTsBlock( case TEXT: case STRING: case BLOB: + case OBJECT: tsBlockBuilder .getColumnBuilder(j) .writeBinary( diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java index 3157b6c98c42..f9c3fe481ac7 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java @@ -70,6 +70,8 @@ public class IoTDBJDBCResultSet implements ResultSet { + public static final String OBJECT_ERR_MSG = "OBJECT Type only support getString"; + private static final Logger LOGGER = LoggerFactory.getLogger(IoTDBJDBCResultSet.class); protected IoTDBStatement statement; @@ -301,6 +303,15 @@ public InputStream getBinaryStream(String arg0) throws SQLException { @Override public Blob getBlob(int arg0) throws SQLException { try { + final TSDataType dataType = ioTDBRpcDataSet.getDataType(arg0); + if (dataType == null) { + return null; + } + + if (dataType.equals(TSDataType.OBJECT)) { + throw new SQLException(OBJECT_ERR_MSG); + } + Binary binary = ioTDBRpcDataSet.getBinary(arg0); if (ObjectUtils.isNotEmpty(binary)) { return new SerialBlob(binary.getValues()); @@ -314,6 +325,15 @@ public Blob getBlob(int arg0) throws SQLException { @Override public Blob getBlob(String arg0) throws SQLException { try { + final TSDataType dataType = ioTDBRpcDataSet.getDataType(arg0); + if (dataType == null) { + return null; + } + + if (dataType.equals(TSDataType.OBJECT)) { + throw new SQLException(OBJECT_ERR_MSG); + } + Binary binary = ioTDBRpcDataSet.getBinary(arg0); if (ObjectUtils.isNotEmpty(binary)) { return new SerialBlob(binary.getValues()); @@ -360,7 +380,9 @@ public byte[] getBytes(int columnIndex) throws SQLException { return null; } - if (dataType.equals(TSDataType.BLOB)) { + if (dataType.equals(TSDataType.OBJECT)) { + throw new SQLException(OBJECT_ERR_MSG); + } else if (dataType.equals(TSDataType.BLOB)) { Binary binary = ioTDBRpcDataSet.getBinary(columnIndex); return binary == null ? null : binary.getValues(); } else { @@ -379,8 +401,9 @@ public byte[] getBytes(String columnName) throws SQLException { if (dataType == null) { return null; } - - if (dataType.equals(TSDataType.BLOB)) { + if (dataType.equals(TSDataType.OBJECT)) { + throw new SQLException(OBJECT_ERR_MSG); + } else if (dataType.equals(TSDataType.BLOB)) { Binary binary = ioTDBRpcDataSet.getBinary(columnName); return binary == null ? null : binary.getValues(); } else { diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java index 5bba6d0ea718..43064ecdaa9f 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java @@ -184,6 +184,7 @@ public IoTDBJDBCDataSet( case TEXT: case BLOB: case STRING: + case OBJECT: values[i] = null; break; default: @@ -307,6 +308,7 @@ public IoTDBJDBCDataSet( case TEXT: case BLOB: case STRING: + case OBJECT: values[i] = null; break; default: @@ -418,6 +420,7 @@ public void constructOneRow() { case TEXT: case BLOB: case STRING: + case OBJECT: int length = valueBuffer.getInt(); values[i] = ReadWriteIOUtils.readBytes(valueBuffer, length); break; @@ -596,6 +599,8 @@ public String getString(int index, TSDataType tsDataType, byte[][] values) { case TEXT: case STRING: return new String(values[index], StandardCharsets.UTF_8); + case OBJECT: + return BytesUtils.parseObjectByteArrayToString(values[index]); case BLOB: return BytesUtils.parseBlobByteArrayToString(values[index]); case DATE: @@ -634,6 +639,7 @@ public Object getObject(int index, TSDataType tsDataType, byte[][] values) { case TEXT: case STRING: return new String(values[index], StandardCharsets.UTF_8); + case OBJECT: case BLOB: return new Binary(values[index]); case TIMESTAMP: diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java index a40197fac798..940b3460d927 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java @@ -494,6 +494,9 @@ private Object getObjectByTsBlockIndex(int tsBlockColumnIndex) .getColumn(tsBlockColumnIndex) .getBinary(tsBlockIndex) .getStringValue(TSFileConfig.STRING_CHARSET); + case OBJECT: + return BytesUtils.parseObjectByteArrayToString( + curTsBlock.getColumn(tsBlockColumnIndex).getBinary(tsBlockIndex).getValues()); case BLOB: return BytesUtils.parseBlobByteArrayToString( curTsBlock.getColumn(tsBlockColumnIndex).getBinary(tsBlockIndex).getValues()); @@ -554,6 +557,9 @@ private String getString(int index, TSDataType tsDataType) { .getColumn(index) .getBinary(tsBlockIndex) .getStringValue(TSFileConfig.STRING_CHARSET); + case OBJECT: + return BytesUtils.parseObjectByteArrayToString( + curTsBlock.getColumn(index).getBinary(tsBlockIndex).getValues()); case BLOB: return BytesUtils.parseBlobByteArrayToString( curTsBlock.getColumn(index).getBinary(tsBlockIndex).getValues()); diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index 6bd0da063b03..bb533ddbbd57 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -145,6 +145,11 @@ public enum TSStatusCode { PLAN_FAILED_NETWORK_PARTITION(721), CANNOT_FETCH_FI_STATE(722), + // OBJECT + OBJECT_NOT_EXISTS(740), + OBJECT_INSERT_ERROR(741), + OBJECT_READ_ERROR(742), + // Arithmetic NUMERIC_VALUE_OUT_OF_RANGE(750), DIVISION_BY_ZERO(751), diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index 8844f36c247d..572e6792e7c4 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -3074,10 +3074,9 @@ private TSInsertTabletReq genTSInsertTabletReq(Tablet tablet, boolean sorted, bo this.columnEncodersMap .getOrDefault( measurementSchema.getType(), - TSEncoding.valueOf( - TSFileDescriptor.getInstance() - .getConfig() - .getValueEncoder(measurementSchema.getType()))) + TSFileDescriptor.getInstance() + .getConfig() + .getValueEncoder(measurementSchema.getType())) .serialize()); } } else { @@ -3745,6 +3744,7 @@ private Object sortList(Object valueList, TSDataType dataType, Integer[] index) case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] binaryValues = (Binary[]) valueList; Binary[] sortedBinaryValues = new Binary[binaryValues.length]; for (int i = 0; i < index.length; i++) { diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java index a3f7f8e3f26d..a3f01b3e11e1 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionSessionDataSet.java @@ -242,6 +242,7 @@ private static Field generateFieldFromTabletValue( case TEXT: case STRING: case BLOB: + case OBJECT: final Binary binaryValue = new Binary((((Binary[]) value)[index]).getValues()); field.setBinaryV(binaryValue); break; diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java index dd30496090c0..1b5ad290678a 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/util/SessionUtils.java @@ -135,6 +135,7 @@ private static int calOccupationOfOneColumn( case TEXT: case BLOB: case STRING: + case OBJECT: valueOccupation += rowSize * 4; Binary[] binaries = (Binary[]) values[columnIndex]; for (int rowIndex = 0; rowIndex < rowSize; rowIndex++) { @@ -185,6 +186,7 @@ public static int calculateLength(List types, List break; case TEXT: case STRING: + case OBJECT: res += Integer.BYTES; if (values.get(i) instanceof Binary) { res += ((Binary) values.get(i)).getValues().length; @@ -336,6 +338,7 @@ private static void getValueBufferOfDataType( case TEXT: case STRING: case BLOB: + case OBJECT: Binary[] binaryValues = (Binary[]) tablet.getValues()[i]; for (int index = 0; index < tablet.getRowSize(); index++) { if (!tablet.isNull(index, i) && binaryValues[index] != null) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 88e8d76001dc..34b69382b8fd 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -316,6 +316,8 @@ public class ConfigNodeConfig { private long forceWalPeriodForConfigNodeSimpleInMs = 100; + private boolean restrictObjectLimit = false; + public ConfigNodeConfig() { // empty constructor } @@ -1275,4 +1277,12 @@ public long getFailureDetectorPhiAcceptablePauseInMs() { public void setFailureDetectorPhiAcceptablePauseInMs(long failureDetectorPhiAcceptablePauseInMs) { this.failureDetectorPhiAcceptablePauseInMs = failureDetectorPhiAcceptablePauseInMs; } + + public boolean getRestrictObjectLimit() { + return restrictObjectLimit; + } + + public void setRestrictObjectLimit(boolean restrictObjectLimit) { + this.restrictObjectLimit = restrictObjectLimit; + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java index 0ea7a278732e..0d3968d3f7a2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java @@ -368,6 +368,11 @@ private void loadProperties(TrimProperties properties) throws BadNodeUrlExceptio readConsistencyLevel)); } + conf.setRestrictObjectLimit( + Boolean.parseBoolean( + properties.getProperty( + "restrict_object_limit", String.valueOf(conf.getRestrictObjectLimit())))); + // commons commonDescriptor.loadCommonProps(properties); commonDescriptor.initCommonConfigDir(conf.getSystemDir()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java index 529f15d06cd4..9cf6b2dac963 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java @@ -213,6 +213,16 @@ public static void checkSystemProperties() throws IOException { COMMON_CONFIG.setEnableGrantOption(enableGrantOption); } } + + if (systemProperties.getProperty("restrict_object_limit", null) != null) { + boolean restrictObjectLimit = + Boolean.parseBoolean(systemProperties.getProperty("restrict_object_limit")); + if (restrictObjectLimit != conf.getRestrictObjectLimit()) { + LOGGER.warn( + format, "restrict_object_limit", conf.getRestrictObjectLimit(), restrictObjectLimit); + conf.setRestrictObjectLimit(restrictObjectLimit); + } + } } /** @@ -286,6 +296,8 @@ public static void storeSystemParameters() throws IOException { "tag_attribute_total_size", String.valueOf(COMMON_CONFIG.getTagAttributeTotalSize())); systemProperties.setProperty( "enable_grant_option", String.valueOf(COMMON_CONFIG.getEnableGrantOption())); + systemProperties.setProperty( + "restrict_object_limit", String.valueOf(conf.getRestrictObjectLimit())); systemPropertiesHandler.overwrite(systemProperties); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index e8083b79b68a..234227286df3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -179,6 +179,7 @@ private void setGlobalConfig(ConfigurationResp dataSet) { globalConfig.setSchemaEngineMode(commonConfig.getSchemaEngineMode()); globalConfig.setTagAttributeTotalSize(commonConfig.getTagAttributeTotalSize()); globalConfig.setEnableGrantOption(commonConfig.getEnableGrantOption()); + globalConfig.setRestrictObjectLimit(configNodeConfig.getRestrictObjectLimit()); dataSet.setGlobalConfig(globalConfig); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 9a04d9e9e59b..a192511ffff0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -111,6 +111,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.annotations.TableModel; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; @@ -124,6 +125,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @@ -1355,9 +1357,14 @@ public synchronized Pair tableColumnCheckForColumnExtension( columnSchemaList.stream() .map(TsTableColumnSchema::getColumnName) .collect(Collectors.joining(", "))); + + final AtomicBoolean hasObject = new AtomicBoolean(false); columnSchemaList.removeIf( columnSchema -> { if (Objects.isNull(originalTable.getColumnSchema(columnSchema.getColumnName()))) { + if (columnSchema.getDataType().equals(TSDataType.OBJECT)) { + hasObject.set(true); + } expandedTable.addColumnSchema(columnSchema); return false; } @@ -1367,7 +1374,11 @@ public synchronized Pair tableColumnCheckForColumnExtension( if (columnSchemaList.isEmpty()) { return new Pair<>(RpcUtils.getStatus(TSStatusCode.COLUMN_ALREADY_EXISTS, errorMsg), null); } - return new Pair<>(RpcUtils.SUCCESS_STATUS, expandedTable); + + if (hasObject.get()) { + expandedTable.checkTableNameAndObjectNames4Object(); + } + return new Pair<>(StatusUtils.OK, expandedTable); } public synchronized Pair tableColumnCheckForColumnRenaming( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java index 4215585b4995..373cec94a663 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeMemoryConfig.java @@ -64,8 +64,7 @@ public class DataNodeMemoryConfig { private int queryThreadCount = Runtime.getRuntime().availableProcessors(); /** Max bytes of each FragmentInstance for DataExchange */ - private long maxBytesPerFragmentInstance = - Runtime.getRuntime().maxMemory() * 3 / 10 * 200 / 1001 / queryThreadCount; + private long maxBytesPerFragmentInstance = Runtime.getRuntime().maxMemory() * 3 / 10 * 200 / 1001; /** The memory manager of on heap */ private MemoryManager onHeapMemoryManager; @@ -487,7 +486,7 @@ private void initQueryEngineMemoryAllocate( operatorsMemorySize += partForOperators; } // set max bytes per fragment instance - setMaxBytesPerFragmentInstance(dataExchangeMemorySize / getQueryThreadCount()); + setMaxBytesPerFragmentInstance(dataExchangeMemorySize); bloomFilterCacheMemoryManager = queryEngineMemoryManager.getOrCreateMemoryManager( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 9f67ab268c2c..6f861119d397 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1207,6 +1207,10 @@ public class IoTDBConfig { new ConcurrentHashMap<>( Collections.singletonMap("root.__audit", new EncryptParameter("UNENCRYPTED", null))); + private long maxObjectSizeInByte = 4 * 1024 * 1024 * 1024L; + + private boolean restrictObjectLimit = false; + IoTDBConfig() {} public int getMaxLogEntriesNumPerBatch() { @@ -4326,4 +4330,20 @@ public void setPasswordLockTimeMinutes(int passwordLockTimeMinutes) { public ConcurrentHashMap getTSFileDBToEncryptMap() { return tsFileDBToEncryptMap; } + + public long getMaxObjectSizeInByte() { + return maxObjectSizeInByte; + } + + public void setMaxObjectSizeInByte(long maxObjectSizeInByte) { + this.maxObjectSizeInByte = maxObjectSizeInByte; + } + + public boolean getRestrictObjectLimit() { + return restrictObjectLimit; + } + + public void setRestrictObjectLimit(boolean restrictObjectLimit) { + this.restrictObjectLimit = restrictObjectLimit; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 2fb7f7efdf0a..a039fc617745 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -900,10 +900,17 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException } conf.setExtPipeDir(properties.getProperty("ext_pipe_dir", conf.getExtPipeDir()).trim()); + conf.setPipeTaskThreadCount( Integer.parseInt( properties.getProperty( "pipe_task_thread_count", Integer.toString(conf.getPipeTaskThreadCount()).trim()))); + + conf.setMaxObjectSizeInByte( + Long.parseLong( + properties.getProperty( + "max_object_file_size_in_byte", String.valueOf(conf.getMaxObjectSizeInByte())))); + // At the same time, set TSFileConfig List fsTypes = new ArrayList<>(); fsTypes.add(FSType.LOCAL); @@ -2170,6 +2177,10 @@ public synchronized void loadHotModifiedProps(TrimProperties properties) "include_null_value_in_write_throughput_metric", ConfigurationFileUtils.getConfigurationDefaultValue( "include_null_value_in_write_throughput_metric")))); + conf.setMaxObjectSizeInByte( + Long.parseLong( + properties.getProperty( + "max_object_file_size_in_byte", String.valueOf(conf.getMaxObjectSizeInByte())))); } catch (Exception e) { if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); @@ -2729,6 +2740,7 @@ public TSEncoding getDefaultEncodingByType(TSDataType dataType) { return conf.getDefaultDoubleEncoding(); case STRING: case BLOB: + case OBJECT: case TEXT: default: return conf.getDefaultTextEncoding(); @@ -2740,6 +2752,7 @@ public void loadGlobalConfig(TGlobalConfig globalConfig) { conf.setSeriesPartitionExecutorClass(globalConfig.getSeriesPartitionExecutorClass()); conf.setSeriesPartitionSlotNum(globalConfig.getSeriesPartitionSlotNum()); conf.setReadConsistencyLevel(globalConfig.getReadConsistencyLevel()); + conf.setRestrictObjectLimit(globalConfig.isRestrictObjectLimit()); } public void loadRatisConfig(TRatisConfig ratisConfig) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java index cdce05b46c0b..0ae796f3913b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/dataregion/DataExecutionVisitor.java @@ -39,6 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; @@ -292,4 +293,16 @@ public TSStatus visitPipeEnrichedDeleteDataNode( node.getDeleteDataNode().markAsGeneratedByPipe(); return node.getDeleteDataNode().accept(this, context); } + + @Override + public TSStatus visitWriteObjectFile(ObjectNode node, DataRegion dataRegion) { + try { + dataRegion.writeObject(node); + dataRegion.insertSeparatorToWAL(); + return StatusUtils.OK; + } catch (final Exception e) { + LOGGER.error("Error in executing plan node: {}", node, e); + return RpcUtils.getStatus(TSStatusCode.OBJECT_INSERT_ERROR.getStatusCode(), e.getMessage()); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java index d16c404e4226..10a72a0d9c5d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/aggregate/AggregateProcessor.java @@ -498,6 +498,7 @@ private Map> processRow( timestamp, row.getString(index), outputMinReportIntervalMilliseconds); break; case BLOB: + case OBJECT: result = state.updateWindows( timestamp, row.getBinary(index), outputMinReportIntervalMilliseconds); @@ -685,6 +686,7 @@ public void collectWindowOutputs( break; case TEXT: case BLOB: + case OBJECT: case STRING: valueColumns[columnIndex] = new Binary[distinctOutputs.size()]; break; @@ -734,6 +736,7 @@ public void collectWindowOutputs( TSFileConfig.STRING_CHARSET); break; case BLOB: + case OBJECT: ((Binary[]) valueColumns[columnIndex])[rowIndex] = (Binary) aggregatedResults.get(columnNameStringList[columnIndex]).getRight(); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java index 6e17d6376814..7f1d7357b02e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/InsertNodeMemoryEstimator.java @@ -602,6 +602,7 @@ public static long sizeOfColumns( case STRING: case TEXT: case BLOB: + case OBJECT: { size += RamUsageEstimator.sizeOf((Binary[]) columns[i]); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcda/OpcDaServerHandle.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcda/OpcDaServerHandle.java index 8ca78a2da554..1ba0a8841d2f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcda/OpcDaServerHandle.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/protocol/opcda/OpcDaServerHandle.java @@ -318,6 +318,7 @@ private short convertTsDataType2VariantType(final TSDataType dataType) { // Note that "Variant" does not support "VT_BLOB" data, and not all the DA server // support this, thus we use "VT_BSTR" to substitute case BLOB: + case OBJECT: return Variant.VT_BSTR; default: throw new UnSupportedDataTypeException("UnSupported dataType " + dataType); @@ -354,6 +355,7 @@ private Variant.VARIANT getTabletObjectValue4Opc( case TEXT: case STRING: case BLOB: + case OBJECT: bstr = OleAuto.INSTANCE.SysAllocString(((Binary[]) column)[rowIndex].toString()); value.setValue(Variant.VT_BSTR, bstr); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/PipeInsertEventSorter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/PipeInsertEventSorter.java index 817df1c06e04..46a3fc6df942 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/PipeInsertEventSorter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/util/sorter/PipeInsertEventSorter.java @@ -185,6 +185,7 @@ protected Object reorderValueListAndBitMap( return deDuplicatedDoubleValues; case TEXT: case BLOB: + case OBJECT: case STRING: final Binary[] binaryValues = (Binary[]) valueList; final Binary[] deDuplicatedBinaryValues = new Binary[binaryValues.length]; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index cb5347e51f2b..167a1fa914fd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -890,7 +890,7 @@ private List executeGroupByQueryInternal( IMeasurementSchema measurementSchema = new MeasurementSchema(measurement, dataType); AbstractSeriesAggregationScanOperator operator; boolean canUseStatistics = - !TSDataType.BLOB.equals(dataType) + (!TSDataType.BLOB.equals(dataType) && !TSDataType.OBJECT.equals(dataType)) || (!TAggregationType.LAST_VALUE.equals(aggregationType) && !TAggregationType.FIRST_VALUE.equals(aggregationType)); IFullPath path; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index dcc62fcd3c8f..8f1b97b71964 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -201,6 +201,7 @@ import org.apache.iotdb.db.trigger.executor.TriggerExecutor; import org.apache.iotdb.db.trigger.executor.TriggerFireResult; import org.apache.iotdb.db.trigger.service.TriggerManagementService; +import org.apache.iotdb.db.utils.ObjectTypeUtils; import org.apache.iotdb.db.utils.SetThreadName; import org.apache.iotdb.metrics.type.AutoGauge; import org.apache.iotdb.metrics.utils.MetricLevel; @@ -279,6 +280,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaReq; import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaResp; import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaRespExceptionMessage; +import org.apache.iotdb.mpp.rpc.thrift.TReadObjectReq; import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeReq; import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeResp; import org.apache.iotdb.mpp.rpc.thrift.TRegionMigrateResult; @@ -3051,6 +3053,12 @@ public TSStatus writeAuditLog(TAuditLogReq req) { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } + @Override + public ByteBuffer readObject(TReadObjectReq req) { + return ObjectTypeUtils.readObjectContent( + req.getRelativePath(), req.getOffset(), req.getSize(), false); + } + public void handleClientExit() { // Do nothing } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java index c61d769bcb3a..fb7938309af0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AccumulatorFactory.java @@ -163,6 +163,7 @@ private static Accumulator createModeAccumulator(TSDataType tsDataType) { return new FloatModeAccumulator(); case DOUBLE: return new DoubleModeAccumulator(); + case OBJECT: default: throw new IllegalArgumentException("Unknown data type: " + tsDataType); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java index c6d1baa33830..6549a7a6748e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/AvgAccumulator.java @@ -57,6 +57,7 @@ public void addInput(Column[] columns, BitMap bitMap) { return; case TEXT: case BLOB: + case OBJECT: case STRING: case BOOLEAN: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java index 76a42b41c718..ff54a48ce5e7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/ExtremeAccumulator.java @@ -58,6 +58,7 @@ public void addInput(Column[] columns, BitMap bitMap) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -90,6 +91,7 @@ public void addIntermediate(Column[] partialResult) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -124,6 +126,7 @@ public void addStatistics(Statistics statistics) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -155,6 +158,7 @@ public void setFinal(Column finalResult) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -188,6 +192,7 @@ public void outputIntermediate(ColumnBuilder[] columnBuilders) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -219,6 +224,7 @@ public void outputFinal(ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java index 3bc2e0f9c896..640008f0a5fd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/FirstValueAccumulator.java @@ -63,6 +63,7 @@ public void addInput(Column[] columns, BitMap bitMap) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(columns, bitMap); return; case BOOLEAN: @@ -98,6 +99,7 @@ public void addIntermediate(Column[] partialResult) { break; case TEXT: case BLOB: + case OBJECT: case STRING: updateBinaryFirstValue(partialResult[0].getBinary(0), partialResult[1].getLong(0)); break; @@ -132,6 +134,7 @@ public void addStatistics(Statistics statistics) { break; case TEXT: case BLOB: + case OBJECT: case STRING: updateBinaryFirstValue((Binary) statistics.getFirstValue(), statistics.getStartTime()); break; @@ -167,6 +170,7 @@ public void setFinal(Column finalResult) { break; case TEXT: case BLOB: + case OBJECT: case STRING: firstValue.setBinary(finalResult.getBinary(0)); break; @@ -206,6 +210,7 @@ public void outputIntermediate(ColumnBuilder[] columnBuilders) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilders[0].writeBinary(firstValue.getBinary()); break; @@ -242,6 +247,7 @@ public void outputFinal(ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(firstValue.getBinary()); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java index 603f3427860a..36c6ed5f88df 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/LastValueAccumulator.java @@ -63,6 +63,7 @@ public void addInput(Column[] columns, BitMap bitMap) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(columns, bitMap); return; case BOOLEAN: @@ -99,6 +100,7 @@ public void addIntermediate(Column[] partialResult) { case TEXT: case BLOB: case STRING: + case OBJECT: updateBinaryLastValue(partialResult[0].getBinary(0), partialResult[1].getLong(0)); break; case BOOLEAN: @@ -133,6 +135,7 @@ public void addStatistics(Statistics statistics) { case TEXT: case BLOB: case STRING: + case OBJECT: updateBinaryLastValue((Binary) statistics.getLastValue(), statistics.getEndTime()); break; case BOOLEAN: @@ -168,6 +171,7 @@ public void setFinal(Column finalResult) { case TEXT: case BLOB: case STRING: + case OBJECT: lastValue.setBinary(finalResult.getBinary(0)); break; case BOOLEAN: @@ -207,6 +211,7 @@ public void outputIntermediate(ColumnBuilder[] columnBuilders) { case TEXT: case BLOB: case STRING: + case OBJECT: columnBuilders[0].writeBinary(lastValue.getBinary()); break; case BOOLEAN: @@ -243,6 +248,7 @@ public void outputFinal(ColumnBuilder columnBuilder) { case TEXT: case BLOB: case STRING: + case OBJECT: columnBuilder.writeBinary(lastValue.getBinary()); break; case BOOLEAN: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java index a0e4f80120cb..ebbd1aa6dc14 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxMinByBaseAccumulator.java @@ -88,6 +88,7 @@ public void addInput(Column[] column, BitMap bitMap) { case TEXT: case BLOB: case BOOLEAN: + case OBJECT: default: throw new UnSupportedDataTypeException(String.format(UNSUPPORTED_TYPE_MESSAGE, yDataType)); } @@ -302,6 +303,7 @@ private void writeX(ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: columnBuilder.writeBinary(xResult.getBinary()); break; case BOOLEAN: @@ -335,6 +337,7 @@ private void updateX(Column xColumn, int xIndex) { case TEXT: case STRING: case BLOB: + case OBJECT: xResult.setBinary(xColumn.getBinary(xIndex)); break; case BOOLEAN: @@ -385,6 +388,7 @@ private void writeIntermediateToStream( case TEXT: case STRING: case BLOB: + case OBJECT: String content = value.getBinary().toString(); dataOutputStream.writeInt(content.length()); dataOutputStream.writeBytes(content); @@ -441,6 +445,7 @@ private void updateFromBytesIntermediateInput(byte[] bytes) { case TEXT: case BLOB: case BOOLEAN: + case OBJECT: default: throw new UnSupportedDataTypeException(String.format(UNSUPPORTED_TYPE_MESSAGE, yDataType)); } @@ -471,6 +476,7 @@ private void readXFromBytesIntermediateInput( case TEXT: case STRING: case BLOB: + case OBJECT: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; columnBuilder.writeBinary(new Binary(BytesUtils.subBytes(bytes, offset, length))); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java index 0d58de8064f2..d02678bd6225 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MaxValueAccumulator.java @@ -64,6 +64,7 @@ public void addInput(Column[] columns, BitMap bitMap) { return; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -98,6 +99,7 @@ public void addIntermediate(Column[] partialResult) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -130,6 +132,7 @@ public void addStatistics(Statistics statistics) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -164,6 +167,7 @@ public void setFinal(Column finalResult) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -199,6 +203,7 @@ public void outputIntermediate(ColumnBuilder[] columnBuilders) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -232,6 +237,7 @@ public void outputFinal(ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java index 1d9cc59aa17d..fd55ea0f1bc7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/MinValueAccumulator.java @@ -64,6 +64,7 @@ public void addInput(Column[] columns, BitMap bitMap) { return; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -98,6 +99,7 @@ public void addIntermediate(Column[] partialResult) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -130,6 +132,7 @@ public void addStatistics(Statistics statistics) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -164,6 +167,7 @@ public void setFinal(Column finalResult) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -199,6 +203,7 @@ public void outputIntermediate(ColumnBuilder[] columnBuilders) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -232,6 +237,7 @@ public void outputFinal(ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java index 37daf1a84b1e..066d8cd4fc5d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/SumAccumulator.java @@ -57,6 +57,7 @@ public void addInput(Column[] columns, BitMap bitMap) { return; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: case TIMESTAMP: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/VarianceAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/VarianceAccumulator.java index 3242518c3dc2..8d27f1497e06 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/VarianceAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/aggregation/VarianceAccumulator.java @@ -70,6 +70,7 @@ public void addInput(Column[] columns, BitMap bitMap) { return; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case STRING: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java index f45e71110d57..b5a831b90b2d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionWriteExecutor.java @@ -66,6 +66,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; @@ -370,6 +371,19 @@ public RegionExecutionResult visitDeleteData( } } + @Override + public RegionExecutionResult visitWriteObjectFile( + final ObjectNode node, final WritePlanNodeExecutionContext context) { + // data deletion don't need to block data insertion, but there are some creation operation + // require write lock on data region. + context.getRegionWriteValidationRWLock().writeLock().lock(); + try { + return super.visitWriteObjectFile(node, context); + } finally { + context.getRegionWriteValidationRWLock().writeLock().unlock(); + } + } + @Override public RegionExecutionResult visitDeleteTimeseries( final DeleteTimeSeriesNode node, final WritePlanNodeExecutionContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/LocalMemoryManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/LocalMemoryManager.java index 03766ab52762..b6ce8b521100 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/LocalMemoryManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/memory/LocalMemoryManager.java @@ -30,7 +30,6 @@ public class LocalMemoryManager { private final MemoryPool queryPool; public LocalMemoryManager() { - // TODO @spricoder: why this pool is only used for query data exchange queryPool = new MemoryPool( "read", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java index 7c671e7b2749..2df0be4c3394 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationUtil.java @@ -262,6 +262,7 @@ public static long getOutputColumnSizePerLine(TSDataType tsDataType) { return BooleanColumn.SIZE_IN_BYTES_PER_POSITION; case TEXT: case BLOB: + case OBJECT: case STRING: return StatisticsManager.getInstance().getMaxBinarySizeInBytes(); default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java index 4668da6413ed..bf021e5b4f40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TopKOperator.java @@ -306,6 +306,7 @@ private void initResultTsBlock() { case TEXT: case STRING: case BLOB: + case OBJECT: columns[i] = new BinaryColumn( positionCount, @@ -382,6 +383,7 @@ private long getMemoryUsageOfOneMergeSortKey() { case TEXT: case STRING: case BLOB: + case OBJECT: memory += 16; break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java index 6dd24e3795a9..28a80462ea77 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/TransformOperator.java @@ -344,6 +344,7 @@ protected YieldableState collectDataPoint(ColumnBuilder writer, long currentTime break; case TEXT: case BLOB: + case OBJECT: case STRING: writer.writeBinary(valueColumn.getBinary(currentIndex)); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/function/partition/Slice.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/function/partition/Slice.java index b7ff6f3e0fdd..57f04be9e001 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/function/partition/Slice.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/function/partition/Slice.java @@ -19,22 +19,29 @@ package org.apache.iotdb.db.queryengine.execution.operator.process.function.partition; +import org.apache.iotdb.db.utils.ObjectTypeUtils; import org.apache.iotdb.udf.api.relational.access.Record; import org.apache.iotdb.udf.api.type.Type; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.DateUtils; +import java.io.File; import java.time.LocalDate; import java.util.Arrays; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.RecordIterator.OBJECT_ERR_MSG; +import static org.apache.iotdb.udf.api.type.Type.OBJECT; + /** Parts of partition. */ public class Slice { private final Column[] requiredColumns; @@ -166,14 +173,28 @@ public boolean getBoolean(int columnIndex) { @Override public Binary getBinary(int columnIndex) { + Type type = dataTypes.get(columnIndex); + if (type == OBJECT) { + throw new UnsupportedOperationException(OBJECT_ERR_MSG); + } + return getBinarySafely(columnIndex); + } + + public Binary getBinarySafely(int columnIndex) { return originalColumns[columnIndex].getBinary(offset); } @Override public String getString(int columnIndex) { - return originalColumns[columnIndex] - .getBinary(offset) - .getStringValue(TSFileConfig.STRING_CHARSET); + Binary binary = originalColumns[columnIndex].getBinary(offset); + Type type = dataTypes.get(columnIndex); + if (type == OBJECT) { + return BytesUtils.parseObjectByteArrayToString(binary.getValues()); + } else if (type == Type.BLOB) { + return BytesUtils.parseBlobByteArrayToString(binary.getValues()); + } else { + return binary.getStringValue(TSFileConfig.STRING_CHARSET); + } } @Override @@ -183,9 +204,35 @@ public LocalDate getLocalDate(int columnIndex) { @Override public Object getObject(int columnIndex) { + Type type = dataTypes.get(columnIndex); + if (type == OBJECT) { + throw new UnsupportedOperationException(OBJECT_ERR_MSG); + } return originalColumns[columnIndex].getObject(offset); } + @Override + public Optional getObjectFile(int columnIndex) { + if (getDataType(columnIndex) != Type.OBJECT) { + throw new UnsupportedOperationException("current column is not object column"); + } + return ObjectTypeUtils.getObjectPathFromBinary(getBinarySafely(columnIndex)); + } + + @Override + public Binary readObject(int columnIndex, long offset, int length) { + if (getDataType(columnIndex) != Type.OBJECT) { + throw new UnsupportedOperationException("current column is not object column"); + } + Binary binary = getBinarySafely(columnIndex); + return new Binary(ObjectTypeUtils.readObjectContent(binary, offset, length, true).array()); + } + + @Override + public Binary readObject(int columnIndex) { + return readObject(columnIndex, 0L, -1); + } + @Override public Type getDataType(int columnIndex) { return dataTypes.get(columnIndex); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java index 205808fd1207..468e340200af 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/join/merge/MergeSortComparator.java @@ -113,6 +113,7 @@ public static Comparator getComparator(TSDataType dataType, int index, break; case TEXT: case BLOB: + case OBJECT: case STRING: comparator = Comparator.comparing( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/expression/PatternExpressionComputation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/expression/PatternExpressionComputation.java index 6b5032627b21..cb8cbdccbb69 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/expression/PatternExpressionComputation.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/expression/PatternExpressionComputation.java @@ -35,6 +35,7 @@ import org.apache.tsfile.read.common.type.BooleanType; import org.apache.tsfile.read.common.type.DoubleType; import org.apache.tsfile.read.common.type.FloatType; +import org.apache.tsfile.read.common.type.ObjectType; import org.apache.tsfile.read.common.type.Type; import java.util.ArrayList; @@ -161,7 +162,9 @@ private Object getValueFromPartition( return partition.getFloat(channel, position); } else if (type instanceof DoubleType) { return partition.getDouble(channel, position); - } else if (type instanceof AbstractVarcharType || type instanceof BlobType) { + } else if (type instanceof AbstractVarcharType + || type instanceof BlobType + || type instanceof ObjectType) { return partition.getBinary(channel, position); } else { throw new SemanticException("Unsupported type: " + type.getClass().getSimpleName()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java index 90b519d52cd9..91e39753344d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LagFunction.java @@ -105,6 +105,7 @@ private void writeDefaultValue( case TEXT: case STRING: case BLOB: + case OBJECT: builder.writeBinary(partition.getBinary(defaultValChannel, index)); return; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java index e21ff13a861b..c51eac156359 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/function/value/LeadFunction.java @@ -106,6 +106,7 @@ private void writeDefaultValue( case TEXT: case STRING: case BLOB: + case OBJECT: builder.writeBinary(partition.getBinary(defaultValChannel, index)); return; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java index ac6f22588093..73ff0fe60e64 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/window/utils/RowComparator.java @@ -99,6 +99,7 @@ private boolean equal(Column column, TSDataType dataType, int offset1, int offse case STRING: case TEXT: case BLOB: + case OBJECT: Binary bin1 = column.getBinary(offset1); Binary bin2 = column.getBinary(offset2); if (!bin1.equals(bin2)) { @@ -178,6 +179,7 @@ private boolean equal(ColumnList column, TSDataType dataType, int offset1, int o case TEXT: case STRING: case BLOB: + case OBJECT: Binary bin1 = column.getBinary(offset1); Binary bin2 = column.getBinary(offset2); if (!bin1.equals(bin2)) { @@ -242,6 +244,7 @@ public boolean equal(List columns1, int offset1, List columns2, case TEXT: case STRING: case BLOB: + case OBJECT: Binary bin1 = column1.getBinary(offset1); Binary bin2 = column2.getBinary(offset2); if (!bin1.equals(bin2)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java index 233d24bf097c..e2fbb4be4eca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java @@ -1068,6 +1068,7 @@ private void addTimeValuePairToResult(TimeValuePair timeValuePair, TsBlockBuilde break; case TEXT: case BLOB: + case OBJECT: case STRING: builder.getColumnBuilder(0).writeBinary(timeValuePair.getValue().getBinary()); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java index fffd43043d30..999842cece36 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/LastQueryAggTableScanOperator.java @@ -726,6 +726,7 @@ private TsPrimitiveType cloneTsPrimitiveType(TsPrimitiveType originalValue) { return new TsPrimitiveType.TsDouble(originalValue.getDouble()); case TEXT: case BLOB: + case OBJECT: case STRING: return new TsPrimitiveType.TsBinary(originalValue.getBinary()); case VECTOR: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java index d7195cc6dfb5..45617de31d1c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java @@ -391,6 +391,7 @@ public static GroupedAccumulator getGroupedApproxMostFrequentAccumulator(TSDataT return new BinaryGroupedApproxMostFrequentAccumulator(); case BLOB: return new BlobGroupedApproxMostFrequentAccumulator(); + case OBJECT: default: throw new UnSupportedDataTypeException( String.format("Unsupported data type in APPROX_COUNT_DISTINCT Aggregation: %s", type)); @@ -416,6 +417,7 @@ public static TableAccumulator getApproxMostFrequentAccumulator(TSDataType type) return new BinaryApproxMostFrequentAccumulator(); case BLOB: return new BlobApproxMostFrequentAccumulator(); + case OBJECT: default: throw new UnSupportedDataTypeException( String.format("Unsupported data type in APPROX_COUNT_DISTINCT Aggregation: %s", type)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java index b78aa1b0177d..ef31759ee677 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ApproxCountDistinctAccumulator.java @@ -78,6 +78,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(arguments[0], mask, hll); return; case BOOLEAN: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java index 464820e22f4d..7bb03faadeb7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java @@ -73,6 +73,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { return; case TEXT: case BLOB: + case OBJECT: case STRING: case BOOLEAN: case DATE: @@ -101,6 +102,7 @@ public void removeInput(Column[] arguments) { return; case TEXT: case BLOB: + case OBJECT: case STRING: case BOOLEAN: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java index db50a3cb7d70..cc9d3e3354cb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/ExtremeAccumulator.java @@ -68,6 +68,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -100,6 +101,7 @@ public void addIntermediate(Column argument) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -136,6 +138,7 @@ public void addStatistics(Statistics[] statistics) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -168,6 +171,7 @@ public void evaluateIntermediate(ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -200,6 +204,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java index 3e4111fd4540..2ca9b7785ccd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstAccumulator.java @@ -82,6 +82,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(arguments[0], arguments[1], mask); return; case BOOLEAN: @@ -132,6 +133,7 @@ public void addIntermediate(Column argument) { case TEXT: case BLOB: case STRING: + case OBJECT: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); @@ -184,6 +186,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { case TEXT: case BLOB: case STRING: + case OBJECT: columnBuilder.writeBinary(firstValue.getBinary()); break; case BOOLEAN: @@ -225,6 +228,7 @@ public void addStatistics(Statistics[] statistics) { case TEXT: case BLOB: case STRING: + case OBJECT: updateBinaryFirstValue( (Binary) statistics[0].getFirstValue(), statistics[0].getStartTime()); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java index e53bfd0bfd5b..2ca6c201f947 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/FirstByAccumulator.java @@ -104,6 +104,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(arguments[0], arguments[1], arguments[2], mask); return; case BOOLEAN: @@ -164,6 +165,7 @@ public void addIntermediate(Column argument) { break; case TEXT: case BLOB: + case OBJECT: case STRING: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; @@ -219,6 +221,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(xResult.getBinary()); break; @@ -278,6 +281,7 @@ public void addStatistics(Statistics[] statistics) { break; case TEXT: case BLOB: + case OBJECT: case STRING: xResult.setBinary((Binary) statistics[0].getFirstValue()); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java index d805500b118b..40f8e2ef97ab 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastAccumulator.java @@ -92,6 +92,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(arguments[0], arguments[1], mask); return; case BOOLEAN: @@ -141,6 +142,7 @@ public void addIntermediate(Column argument) { break; case TEXT: case BLOB: + case OBJECT: case STRING: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; @@ -193,6 +195,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(lastValue.getBinary()); break; @@ -234,6 +237,7 @@ public void addStatistics(Statistics[] statistics) { break; case TEXT: case BLOB: + case OBJECT: case STRING: updateBinaryLastValue((Binary) statistics[0].getLastValue(), statistics[0].getEndTime()); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java index 700242731573..2ee09380b83a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/LastByAccumulator.java @@ -112,6 +112,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(arguments[0], arguments[1], arguments[2], mask); return; case BOOLEAN: @@ -173,6 +174,7 @@ public void addIntermediate(Column argument) { case TEXT: case BLOB: case STRING: + case OBJECT: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; Binary binaryVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); @@ -228,6 +230,7 @@ public void evaluateFinal(ColumnBuilder columnBuilder) { case TEXT: case BLOB: case STRING: + case OBJECT: columnBuilder.writeBinary(xResult.getBinary()); break; case BOOLEAN: @@ -287,6 +290,7 @@ public void addStatistics(Statistics[] statistics) { case TEXT: case BLOB: case STRING: + case OBJECT: xResult.setBinary((Binary) statistics[0].getLastValue()); break; case BOOLEAN: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/MaskedRecordIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/MaskedRecordIterator.java index 5237fcfd12e6..48095927eda5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/MaskedRecordIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/MaskedRecordIterator.java @@ -19,8 +19,6 @@ package org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation; -import org.apache.iotdb.commons.udf.access.RecordIterator; - import org.apache.tsfile.block.column.Column; import org.apache.tsfile.read.common.type.Type; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/access/RecordIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/RecordIterator.java similarity index 64% rename from iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/access/RecordIterator.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/RecordIterator.java index 3c38383f0b02..234723b229b1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/access/RecordIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/RecordIterator.java @@ -17,23 +17,33 @@ * under the License. */ -package org.apache.iotdb.commons.udf.access; +package org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation; import org.apache.iotdb.commons.udf.utils.UDFDataTypeTransformer; +import org.apache.iotdb.db.utils.ObjectTypeUtils; import org.apache.iotdb.udf.api.relational.access.Record; import org.apache.iotdb.udf.api.type.Type; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.read.common.type.ObjectType; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; import org.apache.tsfile.utils.DateUtils; +import java.io.File; import java.time.LocalDate; import java.util.Iterator; import java.util.List; +import java.util.Optional; + +import static org.apache.tsfile.read.common.type.BlobType.BLOB; public class RecordIterator implements Iterator { + public static final String OBJECT_ERR_MSG = + "OBJECT Type only support getString, getObjectFile, objectLength and readObject"; + private final List childrenColumns; private final List dataTypes; private final int positionCount; @@ -70,7 +80,7 @@ private static class RecordImpl implements Record { private final List childrenColumns; private final List dataTypes; - private int index; + private final int index; private RecordImpl( List childrenColumns, @@ -108,15 +118,28 @@ public boolean getBoolean(int columnIndex) { @Override public Binary getBinary(int columnIndex) { + org.apache.tsfile.read.common.type.Type type = dataTypes.get(columnIndex); + if (type == ObjectType.OBJECT) { + throw new UnsupportedOperationException(OBJECT_ERR_MSG); + } + return getBinarySafely(columnIndex); + } + + private Binary getBinarySafely(int columnIndex) { return childrenColumns.get(columnIndex).getBinary(index); } @Override public String getString(int columnIndex) { - return childrenColumns - .get(columnIndex) - .getBinary(index) - .getStringValue(TSFileConfig.STRING_CHARSET); + Binary binary = childrenColumns.get(columnIndex).getBinary(index); + org.apache.tsfile.read.common.type.Type type = dataTypes.get(columnIndex); + if (type == ObjectType.OBJECT) { + return BytesUtils.parseObjectByteArrayToString(binary.getValues()); + } else if (type == BLOB) { + return BytesUtils.parseBlobByteArrayToString(binary.getValues()); + } else { + return binary.getStringValue(TSFileConfig.STRING_CHARSET); + } } @Override @@ -126,9 +149,35 @@ public LocalDate getLocalDate(int columnIndex) { @Override public Object getObject(int columnIndex) { + org.apache.tsfile.read.common.type.Type type = dataTypes.get(columnIndex); + if (type == ObjectType.OBJECT) { + throw new UnsupportedOperationException(OBJECT_ERR_MSG); + } return childrenColumns.get(columnIndex).getObject(index); } + @Override + public Optional getObjectFile(int columnIndex) { + if (getDataType(columnIndex) != Type.OBJECT) { + throw new UnsupportedOperationException("current column is not object column"); + } + return ObjectTypeUtils.getObjectPathFromBinary(getBinarySafely(columnIndex)); + } + + @Override + public Binary readObject(int columnIndex, long offset, int length) { + if (getDataType(columnIndex) != Type.OBJECT) { + throw new UnsupportedOperationException("current column is not object column"); + } + Binary binary = getBinarySafely(columnIndex); + return new Binary(ObjectTypeUtils.readObjectContent(binary, offset, length, true).array()); + } + + @Override + public Binary readObject(int columnIndex) { + return readObject(columnIndex, 0L, -1); + } + @Override public Type getDataType(int columnIndex) { return UDFDataTypeTransformer.transformReadTypeToUDFDataType(dataTypes.get(columnIndex)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/SumAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/SumAccumulator.java index aaab07aa90b8..6f42e1a9995f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/SumAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/SumAccumulator.java @@ -67,6 +67,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { return; case TEXT: case BLOB: + case OBJECT: case STRING: case BOOLEAN: case DATE: @@ -95,6 +96,7 @@ public void removeInput(Column[] arguments) { return; case TEXT: case BLOB: + case OBJECT: case STRING: case BOOLEAN: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java index c33fd5f703a9..c6002304b107 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableMaxMinByBaseAccumulator.java @@ -85,6 +85,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { case STRING: case TEXT: case BLOB: + case OBJECT: addBinaryInput(arguments, mask); return; case BOOLEAN: @@ -350,6 +351,7 @@ private void writeX(ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: columnBuilder.writeBinary(xResult.getBinary()); break; case BOOLEAN: @@ -383,6 +385,7 @@ private void updateX(Column xColumn, int xIndex) { case TEXT: case STRING: case BLOB: + case OBJECT: xResult.setBinary(xColumn.getBinary(xIndex)); break; case BOOLEAN: @@ -451,6 +454,7 @@ private void updateFromBytesIntermediateInput(byte[] bytes) { case STRING: case TEXT: case BLOB: + case OBJECT: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; Binary binaryMaxVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); @@ -494,6 +498,7 @@ private void readXFromBytesIntermediateInput( case TEXT: case STRING: case BLOB: + case OBJECT: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; columnBuilder.writeBinary(new Binary(BytesUtils.subBytes(bytes, offset, length))); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java index 355acba8aa6f..92ea803f2d0a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableVarianceAccumulator.java @@ -79,6 +79,7 @@ public void addInput(Column[] arguments, AggregationMask mask) { return; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case STRING: @@ -106,6 +107,7 @@ public void removeInput(Column[] arguments) { return; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case STRING: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/UserDefinedAggregateFunctionAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/UserDefinedAggregateFunctionAccumulator.java index 64c0896b3a18..70e1c0c5d443 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/UserDefinedAggregateFunctionAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/UserDefinedAggregateFunctionAccumulator.java @@ -19,7 +19,6 @@ package org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation; -import org.apache.iotdb.commons.udf.access.RecordIterator; import org.apache.iotdb.udf.api.State; import org.apache.iotdb.udf.api.customizer.analysis.AggregateFunctionAnalysis; import org.apache.iotdb.udf.api.relational.AggregateFunction; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java index 37f2f5d912dc..99d5fef12089 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Utils.java @@ -54,6 +54,7 @@ public static void serializeValue( case TEXT: case STRING: case BLOB: + case OBJECT: BytesUtils.intToBytes(value.getBinary().getValues().length, valueBytes, offset); offset += 4; System.arraycopy( @@ -116,6 +117,7 @@ public static int calcTypeSize(TSDataType dataType, TsPrimitiveType value) { return 8; case TEXT: case BLOB: + case OBJECT: case STRING: return 4 + value.getBinary().getValues().length; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java index 2573350a31dd..9fac50c2ad24 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedApproxCountDistinctAccumulator.java @@ -78,6 +78,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], mask, hlls, maxStandardError); return; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java index 2947c258e79e..4e177c352b7d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedAvgAccumulator.java @@ -77,6 +77,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { return; case TEXT: case BLOB: + case OBJECT: case STRING: case BOOLEAN: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java index 8d46ab274f4c..f53ea5931423 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedExtremeAccumulator.java @@ -62,6 +62,7 @@ public GroupedExtremeAccumulator(TSDataType seriesDataType) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -92,6 +93,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: break; case BOOLEAN: break; @@ -124,6 +126,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: default: throw new UnSupportedDataTypeException( @@ -150,6 +153,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -183,6 +187,7 @@ public void addIntermediate(int[] groupIds, Column argument) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -214,6 +219,7 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -245,6 +251,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: @@ -277,6 +284,7 @@ public void reset() { case TEXT: case STRING: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case TIMESTAMP: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java index a1884162c494..b8fe8bbe1b22 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstAccumulator.java @@ -78,6 +78,7 @@ public GroupedFirstAccumulator(TSDataType seriesDataType) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues = new BinaryBigArray(); return; case BOOLEAN: @@ -110,6 +111,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += binaryValues.sizeOf(); break; case BOOLEAN: @@ -144,6 +146,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.ensureCapacity(groupCount); return; case BOOLEAN: @@ -176,6 +179,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], arguments[1], mask); return; case BOOLEAN: @@ -225,6 +229,7 @@ public void addIntermediate(int[] groupIds, Column argument) { break; case TEXT: case BLOB: + case OBJECT: case STRING: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; @@ -276,6 +281,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(binaryValues.get(groupId)); break; @@ -313,6 +319,7 @@ public void reset() { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.reset(); return; case BOOLEAN: @@ -356,6 +363,7 @@ private byte[] serializeTimeWithValue(int groupId) { return bytes; case TEXT: case BLOB: + case OBJECT: case STRING: byte[] values = binaryValues.get(groupId).getValues(); length += Integer.BYTES + values.length; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java index 33ee0f908739..d56ba59ef946 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedFirstByAccumulator.java @@ -87,6 +87,7 @@ public GroupedFirstByAccumulator(TSDataType xDataType, TSDataType yDataType) { break; case TEXT: case BLOB: + case OBJECT: case STRING: xBinaryValues = new BinaryBigArray(); break; @@ -120,6 +121,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += xBinaryValues.sizeOf(); break; case BOOLEAN: @@ -156,6 +158,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: xBinaryValues.ensureCapacity(groupCount); return; case BOOLEAN: @@ -192,6 +195,7 @@ public void reset() { break; case TEXT: case BLOB: + case OBJECT: case STRING: xBinaryValues.reset(); break; @@ -227,6 +231,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], arguments[1], arguments[2], mask); return; case BOOLEAN: @@ -288,6 +293,7 @@ public void addIntermediate(int[] groupIds, Column argument) { break; case TEXT: case BLOB: + case OBJECT: case STRING: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; @@ -343,6 +349,7 @@ private byte[] serializeTimeWithValue(int groupId) { return bytes; case TEXT: case BLOB: + case OBJECT: case STRING: byte[] values = xBinaryValues.get(groupId).getValues(); intToBytes(values.length, bytes, Long.BYTES + 1); @@ -373,6 +380,7 @@ private int calculateValueLength(int groupId) { return Double.BYTES; case TEXT: case BLOB: + case OBJECT: case STRING: return Integer.BYTES + xBinaryValues.get(groupId).getValues().length; case BOOLEAN: @@ -407,6 +415,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(xBinaryValues.get(groupId)); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java index 75cc674ef967..b21a683ab0a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastAccumulator.java @@ -78,6 +78,7 @@ public GroupedLastAccumulator(TSDataType seriesDataType) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues = new BinaryBigArray(); return; case BOOLEAN: @@ -110,6 +111,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += binaryValues.sizeOf(); break; case BOOLEAN: @@ -144,6 +146,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.ensureCapacity(groupCount); return; case BOOLEAN: @@ -176,6 +179,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], arguments[1], mask); return; case BOOLEAN: @@ -225,6 +229,7 @@ public void addIntermediate(int[] groupIds, Column argument) { break; case TEXT: case BLOB: + case OBJECT: case STRING: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; @@ -276,6 +281,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(binaryValues.get(groupId)); break; @@ -313,6 +319,7 @@ public void reset() { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.reset(); return; case BOOLEAN: @@ -356,6 +363,7 @@ private byte[] serializeTimeWithValue(int groupId) { return bytes; case TEXT: case BLOB: + case OBJECT: case STRING: byte[] values = binaryValues.get(groupId).getValues(); length += Integer.BYTES + values.length; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java index 386d4c2f34f0..db48d221f18b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedLastByAccumulator.java @@ -87,6 +87,7 @@ public GroupedLastByAccumulator(TSDataType xDataType, TSDataType yDataType) { break; case TEXT: case BLOB: + case OBJECT: case STRING: xBinaryValues = new BinaryBigArray(); break; @@ -120,6 +121,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += xBinaryValues.sizeOf(); break; case BOOLEAN: @@ -156,6 +158,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: xBinaryValues.ensureCapacity(groupCount); return; case BOOLEAN: @@ -192,6 +195,7 @@ public void reset() { break; case TEXT: case BLOB: + case OBJECT: case STRING: xBinaryValues.reset(); break; @@ -227,6 +231,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], arguments[1], arguments[2], mask); return; case BOOLEAN: @@ -288,6 +293,7 @@ public void addIntermediate(int[] groupIds, Column argument) { break; case TEXT: case BLOB: + case OBJECT: case STRING: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; @@ -343,6 +349,7 @@ private byte[] serializeTimeWithValue(int groupId) { return bytes; case TEXT: case BLOB: + case OBJECT: case STRING: byte[] values = xBinaryValues.get(groupId).getValues(); intToBytes(values.length, bytes, Long.BYTES + 1); @@ -373,6 +380,7 @@ private int calculateValueLength(int groupId) { return Double.BYTES; case TEXT: case BLOB: + case OBJECT: case STRING: return Integer.BYTES + xBinaryValues.get(groupId).getValues().length; case BOOLEAN: @@ -407,6 +415,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(xBinaryValues.get(groupId)); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java index 3b538f0244ba..ed47c96d2513 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxAccumulator.java @@ -69,6 +69,7 @@ public GroupedMaxAccumulator(TSDataType seriesDataType) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues = new BinaryBigArray(); return; case BOOLEAN: @@ -101,6 +102,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += binaryValues.sizeOf(); break; case BOOLEAN: @@ -135,6 +137,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.ensureCapacity(groupCount); return; case BOOLEAN: @@ -167,6 +170,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], mask); return; case BOOLEAN: @@ -204,6 +208,7 @@ public void addIntermediate(int[] groupIds, Column argument) { case TEXT: case BLOB: case STRING: + case OBJECT: updateBinaryValue(groupIds[i], argument.getBinary(i)); break; case BOOLEAN: @@ -240,6 +245,7 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { case TEXT: case BLOB: case STRING: + case OBJECT: columnBuilder.writeBinary(binaryValues.get(groupId)); break; case BOOLEAN: @@ -275,6 +281,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { case TEXT: case BLOB: case STRING: + case OBJECT: columnBuilder.writeBinary(binaryValues.get(groupId)); break; case BOOLEAN: @@ -311,6 +318,7 @@ public void reset() { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.reset(); return; case BOOLEAN: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java index 7cfe93629eae..b9b25f627155 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMaxMinByBaseAccumulator.java @@ -96,6 +96,7 @@ protected GroupedMaxMinByBaseAccumulator(TSDataType xDataType, TSDataType yDataT break; case TEXT: case BLOB: + case OBJECT: case STRING: xBinaryValues = new BinaryBigArray(); break; @@ -124,6 +125,7 @@ protected GroupedMaxMinByBaseAccumulator(TSDataType xDataType, TSDataType yDataT break; case TEXT: case BLOB: + case OBJECT: case STRING: yBinaryValues = new BinaryBigArray(); break; @@ -157,6 +159,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += xBinaryValues.sizeOf(); break; case BOOLEAN: @@ -185,6 +188,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += yBinaryValues.sizeOf(); break; case BOOLEAN: @@ -220,6 +224,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: xBinaryValues.ensureCapacity(groupCount); break; case BOOLEAN: @@ -247,6 +252,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: yBinaryValues.ensureCapacity(groupCount); break; case BOOLEAN: @@ -282,6 +288,7 @@ public void reset() { break; case TEXT: case BLOB: + case OBJECT: case STRING: xBinaryValues.reset(); break; @@ -310,6 +317,7 @@ public void reset() { break; case TEXT: case BLOB: + case OBJECT: case STRING: yBinaryValues.reset(); break; @@ -341,6 +349,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { return; case TEXT: case BLOB: + case OBJECT: case STRING: addBinaryInput(groupIds, arguments, mask); return; @@ -596,6 +605,7 @@ private void writeX(int groupId, ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: columnBuilder.writeBinary(xBinaryValues.get(groupId)); break; case BOOLEAN: @@ -630,6 +640,7 @@ private void updateX(int groupId, Column xColumn, int xIndex) { case TEXT: case STRING: case BLOB: + case OBJECT: xBinaryValues.set(groupId, xColumn.getBinary(xIndex)); break; case BOOLEAN: @@ -692,6 +703,7 @@ private void writeIntermediate( case TEXT: case STRING: case BLOB: + case OBJECT: byte[] values = isX ? xBinaryValues.get(groupId).getValues() : yBinaryValues.get(groupId).getValues(); intToBytes(values.length, bytes, offset); @@ -725,6 +737,7 @@ private int calculateValueLength(int groupId, TSDataType dataType, boolean isX) return Double.BYTES; case TEXT: case BLOB: + case OBJECT: case STRING: return Integer.BYTES + (isX @@ -774,6 +787,7 @@ private void updateFromBytesIntermediateInput(int groupId, byte[] bytes) { case STRING: case TEXT: case BLOB: + case OBJECT: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; Binary binaryMaxVal = new Binary(BytesUtils.subBytes(bytes, offset, length)); @@ -818,6 +832,7 @@ private void readXFromBytesIntermediateInput( case TEXT: case STRING: case BLOB: + case OBJECT: int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; columnBuilder.writeBinary(new Binary(BytesUtils.subBytes(bytes, offset, length))); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java index 1c7206928ff7..8fa9b4d8fa01 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedMinAccumulator.java @@ -69,6 +69,7 @@ public GroupedMinAccumulator(TSDataType seriesDataType) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues = new BinaryBigArray(); return; case BOOLEAN: @@ -101,6 +102,7 @@ public long getEstimatedSize() { case TEXT: case STRING: case BLOB: + case OBJECT: valuesSize += binaryValues.sizeOf(); break; case BOOLEAN: @@ -135,6 +137,7 @@ public void setGroupCount(long groupCount) { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.ensureCapacity(groupCount); return; case BOOLEAN: @@ -167,6 +170,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], mask); return; case BOOLEAN: @@ -203,6 +207,7 @@ public void addIntermediate(int[] groupIds, Column argument) { break; case TEXT: case BLOB: + case OBJECT: case STRING: updateBinaryValue(groupIds[i], argument.getBinary(i)); break; @@ -240,6 +245,7 @@ public void evaluateIntermediate(int groupId, ColumnBuilder columnBuilder) { case STRING: case TEXT: case BLOB: + case OBJECT: columnBuilder.writeBinary(binaryValues.get(groupId)); break; case BOOLEAN: @@ -274,6 +280,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { break; case TEXT: case BLOB: + case OBJECT: case STRING: columnBuilder.writeBinary(binaryValues.get(groupId)); break; @@ -311,6 +318,7 @@ public void reset() { case TEXT: case STRING: case BLOB: + case OBJECT: binaryValues.reset(); return; case BOOLEAN: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java index 697a7d3b44a4..a5bf82c5f409 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedModeAccumulator.java @@ -95,6 +95,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { case TEXT: case STRING: case BLOB: + case OBJECT: addBinaryInput(groupIds, arguments[0], mask); break; default: @@ -166,6 +167,7 @@ public void evaluateFinal(int groupId, ColumnBuilder columnBuilder) { case TEXT: case STRING: case BLOB: + case OBJECT: columnBuilder.writeBinary(maxEntry.getKey().getBinary()); break; default: @@ -270,6 +272,7 @@ private byte[] serializeCountMap(int groupId) { case TEXT: case STRING: case BLOB: + case OBJECT: bytes = new byte [offset @@ -366,6 +369,7 @@ private void deserializeAndMergeCountMap(int groupId, byte[] bytes) { case TEXT: case STRING: case BLOB: + case OBJECT: for (int i = 0; i < size; i++) { int length = BytesUtils.bytesToInt(bytes, offset); offset += Integer.BYTES; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java index 4734e7c36718..0c0325ebd0b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedSumAccumulator.java @@ -70,6 +70,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { return; case TEXT: case BLOB: + case OBJECT: case STRING: case BOOLEAN: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedUserDefinedAggregateAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedUserDefinedAggregateAccumulator.java index b3d24e923aed..9ac6b48db7cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedUserDefinedAggregateAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedUserDefinedAggregateAccumulator.java @@ -19,9 +19,9 @@ package org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped; -import org.apache.iotdb.commons.udf.access.RecordIterator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.AggregationMask; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.MaskedRecordIterator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.RecordIterator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.grouped.array.ObjectBigArray; import org.apache.iotdb.udf.api.State; import org.apache.iotdb.udf.api.relational.AggregateFunction; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java index ea8dead07bdd..cb6123b636f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/grouped/GroupedVarianceAccumulator.java @@ -83,6 +83,7 @@ public void addInput(int[] groupIds, Column[] arguments, AggregationMask mask) { return; case TEXT: case BLOB: + case OBJECT: case BOOLEAN: case DATE: case STRING: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java index 3c3fc6e616aa..ae4975dff9ad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/window/WindowManagerFactory.java @@ -70,6 +70,7 @@ private static VariationWindowManager genEqualEventWindowManager( case BOOLEAN: return new EqualBooleanWindowManager(eventWindowParameter, ascending); case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case DATE: @@ -97,6 +98,7 @@ private static VariationWindowManager genVariationEventWindowManager( case STRING: case BOOLEAN: case BLOB: + case OBJECT: case TEXT: default: throw new UnSupportedDataTypeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java index 9580f440e04f..41279c9b3eb9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.InputLocation; import org.apache.iotdb.db.queryengine.plan.relational.function.arithmetic.AdditionResolver; @@ -163,9 +164,11 @@ import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.Log10ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.LongToBytesColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.LowerColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.ObjectLengthColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RTrim2ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RTrimColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RadiansColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.ReadObjectColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RegexpLike2ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.RegexpLikeColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.Replace2ColumnTransformer; @@ -212,6 +215,8 @@ import org.apache.tsfile.read.common.type.TypeEnum; import org.apache.tsfile.utils.Binary; +import javax.annotation.Nullable; + import java.time.ZoneId; import java.util.ArrayList; import java.util.Arrays; @@ -226,6 +231,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static org.apache.iotdb.db.queryengine.plan.expression.unary.LikeExpression.getEscapeCharacter; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.predicate.PredicatePushIntoMetadataChecker.isStringLiteral; +import static org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl.isBlobType; import static org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl.isCharType; import static org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager.getTSDataType; import static org.apache.iotdb.db.queryengine.plan.relational.type.TypeSignatureTranslator.toTypeSignature; @@ -778,9 +784,11 @@ private ColumnTransformer getFunctionColumnTransformer( if (children.size() == 1) { Type argumentType = first.getType(); if (isCharType(argumentType)) { - return new LengthColumnTransformer(INT32, first); + return new LengthColumnTransformer(INT64, first); + } else if (isBlobType(argumentType)) { + return new BlobLengthColumnTransformer(INT64, first); } else { - return new BlobLengthColumnTransformer(INT32, first); + return new ObjectLengthColumnTransformer(INT64, first); } } } else if (TableBuiltinScalarFunction.UPPER.getFunctionName().equalsIgnoreCase(functionName)) { @@ -1452,6 +1460,29 @@ private ColumnTransformer getFunctionColumnTransformer( this.process(children.get(0), context), this.process(children.get(1), context), this.process(children.get(2), context)); + } else if (TableBuiltinScalarFunction.READ_OBJECT + .getFunctionName() + .equalsIgnoreCase(functionName)) { + ColumnTransformer first = this.process(children.get(0), context); + if (children.size() == 1) { + return new ReadObjectColumnTransformer(BLOB, first, context.fragmentInstanceContext); + } else if (children.size() == 2) { + return new ReadObjectColumnTransformer( + BLOB, + ((LongLiteral) children.get(1)).getParsedValue(), + first, + context.fragmentInstanceContext); + } else { + long offset = ((LongLiteral) children.get(1)).getParsedValue(); + long length = ((LongLiteral) children.get(2)).getParsedValue(); + checkArgument(offset >= 0 && length >= 0); + return new ReadObjectColumnTransformer( + BLOB, + ((LongLiteral) children.get(1)).getParsedValue(), + ((LongLiteral) children.get(2)).getParsedValue(), + first, + context.fragmentInstanceContext); + } } else { // user defined function if (TableUDFUtils.isScalarFunction(functionName)) { @@ -1916,6 +1947,8 @@ public static class Context { private final Metadata metadata; + private final Optional fragmentInstanceContext; + public Context( SessionInfo sessionInfo, List leafList, @@ -1926,7 +1959,8 @@ public Context( List inputDataTypes, int originSize, TypeProvider typeProvider, - Metadata metadata) { + Metadata metadata, + @Nullable FragmentInstanceContext fragmentInstanceContext) { this.sessionInfo = sessionInfo; this.leafList = leafList; this.inputLocations = inputLocations; @@ -1937,6 +1971,7 @@ public Context( this.originSize = originSize; this.typeProvider = typeProvider; this.metadata = metadata; + this.fragmentInstanceContext = Optional.ofNullable(fragmentInstanceContext); } public Type getType(SymbolReference symbolReference) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java index 8c54fd640f88..2274762341b5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/ClusterPartitionFetcher.java @@ -308,6 +308,10 @@ public boolean updateRegionCache(final TRegionRouteReq req) { return partitionCache.updateGroupIdToReplicaSetMap(req.getTimestamp(), req.getRegionRouteMap()); } + public List getRegionReplicaSet(List consensusGroupIds) { + return partitionCache.getRegionReplicaSet(consensusGroupIds); + } + @Override public void invalidAllCache() { partitionCache.invalidAllCache(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index bc52a3a7e8cb..d659f30fc8fe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -27,6 +27,7 @@ import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.auth.entity.User; import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.exception.auth.AccessDeniedException; import org.apache.iotdb.commons.executable.ExecutableManager; import org.apache.iotdb.commons.path.PartialPath; @@ -571,10 +572,12 @@ private Pair parseTable4CreateTableOrView( // TODO: Place the check at statement analyzer boolean hasTimeColumn = false; final Set sourceNameSet = new HashSet<>(); + boolean hasObject = false; for (final ColumnDefinition columnDefinition : node.getElements()) { final TsTableColumnCategory category = columnDefinition.getColumnCategory(); final String columnName = columnDefinition.getName().getValue(); final TSDataType dataType = getDataType(columnDefinition.getType()); + hasObject |= dataType.equals(TSDataType.OBJECT); final String comment = columnDefinition.getComment(); if (checkTimeColumnIdempotent(category, columnName, dataType, comment, table) && !hasTimeColumn) { @@ -603,6 +606,13 @@ private Pair parseTable4CreateTableOrView( } table.addColumnSchema(schema); } + if (hasObject) { + try { + table.checkTableNameAndObjectNames4Object(); + } catch (final MetadataException e) { + throw new SemanticException(e.getMessage(), e.getErrorCode()); + } + } return new Pair<>(database, table); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 4d99a3ed8928..e33cc6154078 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -329,6 +329,7 @@ import org.apache.iotdb.udf.api.relational.ScalarFunction; import org.apache.iotdb.udf.api.relational.TableFunction; import org.apache.iotdb.udf.api.relational.table.specification.ParameterSpecification; +import org.apache.iotdb.udf.api.relational.table.specification.ScalarParameterSpecification; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.SettableFuture; @@ -373,6 +374,7 @@ import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_RESULT_NODES; import static org.apache.iotdb.db.protocol.client.ConfigNodeClient.MSG_RECONNECTION_FAIL; import static org.apache.iotdb.db.utils.constant.SqlConstant.ROOT; +import static org.apache.iotdb.udf.api.type.Type.OBJECT; public class ClusterConfigTaskExecutor implements IConfigTaskExecutor { @@ -678,6 +680,16 @@ public SettableFuture createFunction( + "'.", TSStatusCode.UDF_LOAD_CLASS_ERROR.getStatusCode())); return future; + } else if (checkObjectScalarParameter(specification)) { + future.setException( + new IoTDBException( + "Failed to create function '" + + udfName + + "', because there is an argument with OBJECT type '" + + specification.getName() + + "'.", + TSStatusCode.UDF_LOAD_CLASS_ERROR.getStatusCode())); + return future; } } } @@ -721,6 +733,15 @@ public SettableFuture createFunction( return future; } + private boolean checkObjectScalarParameter(ParameterSpecification parameterSpecification) { + if (parameterSpecification instanceof ScalarParameterSpecification) { + ScalarParameterSpecification scalarParameterSpecification = + (ScalarParameterSpecification) parameterSpecification; + return scalarParameterSpecification.getType() == OBJECT; + } + return false; + } + @Override public SettableFuture dropFunction(Model model, String udfName) { SettableFuture future = SettableFuture.create(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index 581641591837..44be3e69afb7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -1435,6 +1435,7 @@ public static IFill[] getPreviousFill( case TEXT: case STRING: case BLOB: + case OBJECT: previousFill[i] = filter == null ? new BinaryPreviousFill() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index a2bd0dc48150..e699ee417deb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -41,6 +41,7 @@ import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISinkHandle; import org.apache.iotdb.db.queryengine.execution.exchange.sink.ShuffleSinkHandle; import org.apache.iotdb.db.queryengine.execution.exchange.source.ISourceHandle; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceManager; import org.apache.iotdb.db.queryengine.execution.operator.EmptyDataOperator; import org.apache.iotdb.db.queryengine.execution.operator.ExplainAnalyzeOperator; @@ -267,6 +268,7 @@ import org.apache.tsfile.read.common.type.BinaryType; import org.apache.tsfile.read.common.type.BlobType; import org.apache.tsfile.read.common.type.BooleanType; +import org.apache.tsfile.read.common.type.ObjectType; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeFactory; import org.apache.tsfile.read.filter.basic.Filter; @@ -1339,13 +1341,15 @@ private Operator constructFilterAndProjectOperator( ColumnTransformerBuilder visitor = new ColumnTransformerBuilder(); + FragmentInstanceContext fragmentInstanceContext = + context.getDriverContext().getFragmentInstanceContext(); ColumnTransformer filterOutputTransformer = predicate .map( p -> { ColumnTransformerBuilder.Context filterColumnTransformerContext = new ColumnTransformerBuilder.Context( - context.getDriverContext().getFragmentInstanceContext().getSessionInfo(), + fragmentInstanceContext.getSessionInfo(), filterLeafColumnTransformerList, inputLocations, filterExpressionColumnTransformerMap, @@ -1354,7 +1358,8 @@ private Operator constructFilterAndProjectOperator( ImmutableList.of(), 0, context.getTypeProvider(), - metadata); + metadata, + fragmentInstanceContext); return visitor.process(p, filterColumnTransformerContext); }) @@ -1372,7 +1377,7 @@ private Operator constructFilterAndProjectOperator( ColumnTransformerBuilder.Context projectColumnTransformerContext = new ColumnTransformerBuilder.Context( - context.getDriverContext().getFragmentInstanceContext().getSessionInfo(), + fragmentInstanceContext.getSessionInfo(), projectLeafColumnTransformerList, inputLocations, projectExpressionColumnTransformerMap, @@ -1381,7 +1386,8 @@ private Operator constructFilterAndProjectOperator( filterOutputDataTypes, inputLocations.size(), context.getTypeProvider(), - metadata); + metadata, + fragmentInstanceContext); for (Expression expression : projectExpressions) { projectOutputTransformerList.add( @@ -2461,6 +2467,8 @@ public Operator visitTableDeviceQueryCount( // In "count" we have to reuse filter operator per "next" final List filterLeafColumnTransformerList = new ArrayList<>(); + FragmentInstanceContext fragmentInstanceContext = + context.getDriverContext().getFragmentInstanceContext(); return new SchemaCountOperator<>( node.getPlanNodeId(), context @@ -2482,10 +2490,7 @@ public Operator visitTableDeviceQueryCount( .process( node.getTagFuzzyPredicate(), new ColumnTransformerBuilder.Context( - context - .getDriverContext() - .getFragmentInstanceContext() - .getSessionInfo(), + fragmentInstanceContext.getSessionInfo(), filterLeafColumnTransformerList, makeLayout(Collections.singletonList(node)), new HashMap<>(), @@ -2494,7 +2499,8 @@ public Operator visitTableDeviceQueryCount( ImmutableList.of(), 0, context.getTypeProvider(), - metadata)), + metadata, + fragmentInstanceContext)), columnSchemaList, database, table) @@ -3828,6 +3834,7 @@ private boolean[] checkStatisticAndScanOrder( case MAX: case MIN: if (BlobType.BLOB.equals(argumentType) + || ObjectType.OBJECT.equals(argumentType) || BinaryType.TEXT.equals(argumentType) || BooleanType.BOOLEAN.equals(argumentType)) { canUseStatistic = false; @@ -3843,8 +3850,8 @@ private boolean[] checkStatisticAndScanOrder( descendingCount++; } - // first/last/first_by/last_by aggregation with BLOB type can not use statistics - if (BlobType.BLOB.equals(argumentType)) { + // first/last/first_by/last_by aggregation with BLOB or OBJECT type can not use statistics + if (BlobType.BLOB.equals(argumentType) || ObjectType.OBJECT.equals(argumentType)) { canUseStatistic = false; break; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index 56ad6d59ba0e..6ebfa6dcfdc4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -112,6 +112,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; @@ -321,6 +322,7 @@ public enum PlanNodeType { RELATIONAL_INSERT_ROW((short) 2001), RELATIONAL_INSERT_ROWS((short) 2002), RELATIONAL_DELETE_DATA((short) 2003), + OBJECT_FILE_NODE((short) 2004), ; public static final int BYTES = Short.BYTES; @@ -364,6 +366,8 @@ public static PlanNode deserializeFromWAL(DataInputStream stream) throws IOExcep return RelationalInsertRowsNode.deserializeFromWAL(stream); case 2003: return RelationalDeleteDataNode.deserializeFromWAL(stream); + case 2004: + return ObjectNode.deserializeFromWAL(stream); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } @@ -390,6 +394,8 @@ public static PlanNode deserializeFromWAL(ByteBuffer buffer) { return RelationalInsertRowsNode.deserializeFromWAL(buffer); case 2003: return RelationalDeleteDataNode.deserializeFromWAL(buffer); + case 2004: + return ObjectNode.deserialize(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } @@ -717,6 +723,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return RelationalInsertRowsNode.deserialize(buffer); case 2003: return RelationalDeleteDataNode.deserialize(buffer); + case 2004: + return ObjectNode.deserialize(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index ea669491a5f1..4bebb692254a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -116,6 +116,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode; @@ -633,6 +634,10 @@ public R visitDeleteData(RelationalDeleteDataNode node, C context) { return visitPlan(node, context); } + public R visitWriteObjectFile(ObjectNode node, C context) { + return visitPlan(node, context); + } + ///////////////////////////////////////////////////////////////////////////////////////////////// // Pipe Related Node ///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index 04bef0b577c3..760d36f1f5cd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -367,6 +367,7 @@ private void putDataTypesAndValues(ByteBuffer buffer) { case TEXT: case STRING: case BLOB: + case OBJECT: ReadWriteIOUtils.write((Binary) values[i], buffer); break; default: @@ -426,6 +427,7 @@ private void putDataTypesAndValues(DataOutputStream stream) throws IOException { case TEXT: case STRING: case BLOB: + case OBJECT: ReadWriteIOUtils.write((Binary) values[i], stream); break; default: @@ -520,6 +522,7 @@ private void fillDataTypesAndValues(ByteBuffer buffer) { case TEXT: case STRING: case BLOB: + case OBJECT: values[i] = ReadWriteIOUtils.readBinary(buffer); break; default: @@ -589,6 +592,7 @@ private int serializeMeasurementsAndValuesSize() { case TEXT: case STRING: case BLOB: + case OBJECT: size += ReadWriteIOUtils.sizeToWrite((Binary) values[i]); break; default: @@ -668,6 +672,7 @@ private void putDataTypesAndValues(IWALByteBufferView buffer) { case TEXT: case BLOB: case STRING: + case OBJECT: WALWriteUtils.write((Binary) values[i], buffer); break; default: @@ -759,6 +764,7 @@ public void fillDataTypesAndValuesFromWAL(DataInputStream stream) throws IOExcep case TEXT: case STRING: case BLOB: + case OBJECT: values[i] = ReadWriteIOUtils.readBinary(stream); break; default: @@ -849,6 +855,7 @@ public void fillDataTypesAndValuesFromWAL(ByteBuffer buffer) { case TEXT: case STRING: case BLOB: + case OBJECT: values[i] = ReadWriteIOUtils.readBinary(buffer); break; default: @@ -889,7 +896,9 @@ public R accept(PlanVisitor visitor, C context) { } public TimeValuePair composeTimeValuePair(int columnIndex) { - if (columnIndex >= values.length || Objects.isNull(dataTypes[columnIndex])) { + if (columnIndex >= values.length + || Objects.isNull(dataTypes[columnIndex]) + || dataTypes[columnIndex] == TSDataType.OBJECT) { return null; } Object value = values[columnIndex]; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index edbc262971a6..3d3c25c0613a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -94,10 +94,26 @@ public class InsertTabletNode extends InsertNode implements WALEntryValue { // proper positions. protected List range; + private Boolean shouldCheckTTL; + public InsertTabletNode(PlanNodeId id) { super(id); } + public boolean shouldCheckTTL() { + if (shouldCheckTTL != null) { + return shouldCheckTTL; + } + shouldCheckTTL = true; + for (MeasurementSchema measurementSchema : measurementSchemas) { + if (measurementSchema != null && measurementSchema.getType() == TSDataType.OBJECT) { + shouldCheckTTL = false; + break; + } + } + return shouldCheckTTL; + } + @Override public InsertNode mergeInsertNode(List insertNodes) { List index = new ArrayList<>(); @@ -219,7 +235,6 @@ public List splitByPartition(IAnalysis analysis) { final Map deviceIDSplitInfoMap = collectSplitRanges(); final Map> splitMap = splitByReplicaSet(deviceIDSplitInfoMap, analysis); - return doSplit(splitMap); } @@ -289,7 +304,7 @@ protected Map> splitByReplicaSet( return splitMap; } - private List doSplit(Map> splitMap) { + protected List doSplit(Map> splitMap) { List result = new ArrayList<>(); if (splitMap.size() == 1) { @@ -326,7 +341,7 @@ protected InsertTabletNode getEmptySplit(int count) { subTimes.length); } - private WritePlanNode generateOneSplit(Map.Entry> entry) { + protected WritePlanNode generateOneSplit(Map.Entry> entry) { List locs; // generate a new times and values locs = entry.getValue(); @@ -387,6 +402,7 @@ protected Object[] initTabletValues(int columnSize, int rowSize, TSDataType[] da case TEXT: case BLOB: case STRING: + case OBJECT: values[i] = new Binary[rowSize]; break; case FLOAT: @@ -640,6 +656,7 @@ private void serializeColumn(TSDataType dataType, Object column, ByteBuffer buff case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] binaryValues = (Binary[]) column; for (int j = 0; j < rowCount; j++) { if (binaryValues[j] != null && binaryValues[j].getValues() != null) { @@ -692,6 +709,7 @@ private void serializeColumn(TSDataType dataType, Object column, DataOutputStrea case STRING: case TEXT: case BLOB: + case OBJECT: Binary[] binaryValues = (Binary[]) column; for (int j = 0; j < rowCount; j++) { if (binaryValues[j] != null && binaryValues[j].getValues() != null) { @@ -833,6 +851,7 @@ private int getColumnSize(TSDataType dataType, Object column, int start, int end case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] binaryValues = (Binary[]) column; for (int j = start; j < end; j++) { size += ReadWriteIOUtils.sizeToWrite(binaryValues[j]); @@ -964,6 +983,7 @@ private void serializeColumn( case STRING: case TEXT: case BLOB: + case OBJECT: Binary[] binaryValues = (Binary[]) column; for (int j = start; j < end; j++) { if (binaryValues[j] != null && binaryValues[j].getValues() != null) { @@ -1127,6 +1147,7 @@ private boolean equals(Object[] columns) { case TEXT: case BLOB: case STRING: + case OBJECT: if (!Arrays.equals((Binary[]) this.columns[i], (Binary[]) columns[i])) { return false; } @@ -1199,6 +1220,8 @@ public TimeValuePair composeLastTimeValuePair( Binary[] binaryValues = (Binary[]) columns[measurementIndex]; value = new TsPrimitiveType.TsBinary(binaryValues[lastIdx]); break; + case OBJECT: + return null; default: throw new UnSupportedDataTypeException( String.format(DATATYPE_UNSUPPORTED, dataTypes[measurementIndex])); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/ObjectNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/ObjectNode.java new file mode 100644 index 000000000000..4ec37e46bf25 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/ObjectNode.java @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write; + +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.consensus.index.ProgressIndex; +import org.apache.iotdb.commons.exception.ObjectFileNotExist; +import org.apache.iotdb.commons.exception.runtime.SerializationRunTimeException; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.storageengine.dataregion.IObjectPath; +import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryType; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; +import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; + +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Optional; + +public class ObjectNode extends SearchNode implements WALEntryValue { + + private static final Logger LOGGER = LoggerFactory.getLogger(ObjectNode.class); + + private final boolean isEOF; + + private final long offset; + + private byte[] content; + + private IObjectPath filePath; + + private final int contentLength; + + private TRegionReplicaSet dataRegionReplicaSet; + + private boolean isGeneratedByRemoteConsensusLeader; + + public ObjectNode(boolean isEOF, long offset, byte[] content, IObjectPath filePath) { + super(new PlanNodeId("")); + this.isEOF = isEOF; + this.offset = offset; + this.filePath = filePath; + this.content = content; + this.contentLength = content.length; + } + + public ObjectNode(boolean isEOF, long offset, int contentLength, IObjectPath filePath) { + super(new PlanNodeId("")); + this.isEOF = isEOF; + this.offset = offset; + this.filePath = filePath; + this.contentLength = contentLength; + } + + public boolean isEOF() { + return isEOF; + } + + public byte[] getContent() { + return content; + } + + public long getOffset() { + return offset; + } + + public void setFilePath(IObjectPath filePath) { + this.filePath = filePath; + } + + public String getFilePathString() { + return filePath.toString(); + } + + @Override + public void serializeToWAL(IWALByteBufferView buffer) { + buffer.putShort(getType().getNodeType()); + buffer.putLong(searchIndex); + buffer.put((byte) (isEOF ? 1 : 0)); + buffer.putLong(offset); + try { + filePath.serialize(buffer); + } catch (IOException e) { + throw new RuntimeException(e); + } + buffer.putInt(content.length); + } + + @Override + public int serializedSize() { + return Short.BYTES + + Long.BYTES + + Byte.BYTES + + Long.BYTES + + Integer.BYTES + + filePath.getSerializedSize(); + } + + public static ObjectNode deserializeFromWAL(DataInputStream stream) throws IOException { + long searchIndex = stream.readLong(); + boolean isEOF = stream.readByte() == 1; + long offset = stream.readLong(); + IObjectPath filePath = IObjectPath.getDeserializer().deserializeFrom(stream); + int contentLength = stream.readInt(); + ObjectNode objectNode = new ObjectNode(isEOF, offset, contentLength, filePath); + objectNode.setSearchIndex(searchIndex); + return objectNode; + } + + public static ObjectNode deserializeFromWAL(ByteBuffer buffer) { + long searchIndex = buffer.getLong(); + boolean isEOF = buffer.get() == 1; + long offset = buffer.getLong(); + IObjectPath filePath = IObjectPath.getDeserializer().deserializeFrom(buffer); + Optional objectFile = + TierManager.getInstance().getAbsoluteObjectFilePath(filePath.toString()); + int contentLength = buffer.getInt(); + byte[] contents = new byte[contentLength]; + if (objectFile.isPresent()) { + try (RandomAccessFile raf = new RandomAccessFile(objectFile.get(), "r")) { + raf.seek(offset); + raf.read(contents); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + throw new ObjectFileNotExist(filePath.toString()); + } + + ObjectNode objectNode = new ObjectNode(isEOF, offset, contents, filePath); + objectNode.setSearchIndex(searchIndex); + return objectNode; + } + + public static ObjectNode deserialize(ByteBuffer byteBuffer) { + boolean isEoF = ReadWriteIOUtils.readBool(byteBuffer); + long offset = ReadWriteIOUtils.readLong(byteBuffer); + IObjectPath filePath = IObjectPath.getDeserializer().deserializeFrom(byteBuffer); + int contentLength = ReadWriteIOUtils.readInt(byteBuffer); + byte[] content = ReadWriteIOUtils.readBytes(byteBuffer, contentLength); + return new ObjectNode(isEoF, offset, content, filePath); + } + + @Override + public SearchNode merge(List searchNodes) { + if (searchNodes.size() == 1) { + return searchNodes.get(0); + } + throw new UnsupportedOperationException("Merge is not supported"); + } + + @Override + public ProgressIndex getProgressIndex() { + return null; + } + + @Override + public void setProgressIndex(ProgressIndex progressIndex) {} + + @Override + public List splitByPartition(IAnalysis analysis) { + return null; + } + + @Override + public TRegionReplicaSet getRegionReplicaSet() { + return dataRegionReplicaSet; + } + + public void setDataRegionReplicaSet(TRegionReplicaSet dataRegionReplicaSet) { + this.dataRegionReplicaSet = dataRegionReplicaSet; + } + + @Override + public List getChildren() { + return null; + } + + @Override + public void addChild(PlanNode child) {} + + @Override + public PlanNode clone() { + return null; + } + + @Override + public int allowedChildCount() { + return NO_CHILD_ALLOWED; + } + + @Override + public List getOutputColumnNames() { + return null; + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + getType().serialize(byteBuffer); + ReadWriteIOUtils.write(isEOF, byteBuffer); + ReadWriteIOUtils.write(offset, byteBuffer); + filePath.serialize(byteBuffer); + ReadWriteIOUtils.write(contentLength, byteBuffer); + byteBuffer.put(content); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + getType().serialize(stream); + ReadWriteIOUtils.write(isEOF, stream); + ReadWriteIOUtils.write(offset, stream); + filePath.serialize(stream); + ReadWriteIOUtils.write(contentLength, stream); + stream.write(content); + } + + public ByteBuffer serialize() { + try (PublicBAOS byteArrayOutputStream = new PublicBAOS(); + DataOutputStream stream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(WALEntryType.OBJECT_FILE_NODE.getCode(), stream); + ReadWriteIOUtils.write((long) TsFileProcessor.MEMTABLE_NOT_EXIST, stream); + ReadWriteIOUtils.write(getType().getNodeType(), stream); + byte[] contents = new byte[contentLength]; + boolean readSuccess = false; + for (int i = 0; i < 2; i++) { + Optional objectFile = + TierManager.getInstance().getAbsoluteObjectFilePath(filePath.toString()); + if (objectFile.isPresent()) { + try { + readContentFromFile(objectFile.get(), contents); + readSuccess = true; + } catch (IOException e) { + LOGGER.error("Error when read object file {}.", objectFile.get(), e); + } + if (readSuccess) { + break; + } + } + Optional objectTmpFile = + TierManager.getInstance().getAbsoluteObjectFilePath(filePath + ".tmp"); + if (objectTmpFile.isPresent()) { + try { + readContentFromFile(objectTmpFile.get(), contents); + readSuccess = true; + } catch (IOException e) { + LOGGER.error("Error when read tmp object file {}.", objectTmpFile.get(), e); + } + if (readSuccess) { + break; + } + } + } + ReadWriteIOUtils.write(readSuccess && isEOF, stream); + ReadWriteIOUtils.write(offset, stream); + filePath.serialize(stream); + ReadWriteIOUtils.write(contentLength, stream); + stream.write(contents); + return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } catch (IOException e) { + throw new SerializationRunTimeException(e); + } + } + + private void readContentFromFile(File file, byte[] contents) throws IOException { + try (RandomAccessFile raf = new RandomAccessFile(file, "r")) { + raf.seek(offset); + raf.read(contents); + } + } + + @Override + public PlanNodeType getType() { + return PlanNodeType.OBJECT_FILE_NODE; + } + + @Override + public long getMemorySize() { + return contentLength; + } + + @Override + public void markAsGeneratedByRemoteConsensusLeader() { + isGeneratedByRemoteConsensusLeader = true; + } + + public boolean isGeneratedByRemoteConsensusLeader() { + return isGeneratedByRemoteConsensusLeader; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitWriteObjectFile(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java index 77020d9220dc..6eeb3d7322fe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertRowsNode.java @@ -27,9 +27,13 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; +import org.apache.iotdb.db.storageengine.dataregion.IObjectPath; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.ReadWriteIOUtils; import java.io.DataInputStream; import java.io.IOException; @@ -39,6 +43,8 @@ import java.util.List; import java.util.Map; +import static org.apache.iotdb.db.utils.ObjectTypeUtils.generateObjectBinary; + public class RelationalInsertRowsNode extends InsertRowsNode { // deviceId cache for Table-view insertion private IDeviceID[] deviceIDs; @@ -159,6 +165,7 @@ public String getTableName() { @Override public List splitByPartition(IAnalysis analysis) { + List writePlanNodeList = new ArrayList<>(); Map splitMap = new HashMap<>(); List redirectInfo = new ArrayList<>(); for (int i = 0; i < getInsertRowNodeList().size(); i++) { @@ -172,6 +179,9 @@ public List splitByPartition(IAnalysis analysis) { insertRowNode.getDeviceID(), TimePartitionUtils.getTimePartitionSlot(insertRowNode.getTime()), analysis.getDatabaseName()); + // handle object type + handleObjectValue(insertRowNode, dataRegionReplicaSet, writePlanNodeList); + // Collect redirectInfo redirectInfo.add(dataRegionReplicaSet.getDataNodeLocations().get(0).getClientRpcEndPoint()); RelationalInsertRowsNode tmpNode = splitMap.get(dataRegionReplicaSet); @@ -185,8 +195,44 @@ public List splitByPartition(IAnalysis analysis) { } } analysis.setRedirectNodeList(redirectInfo); + writePlanNodeList.addAll(splitMap.values()); + + return writePlanNodeList; + } - return new ArrayList<>(splitMap.values()); + private void handleObjectValue( + InsertRowNode insertRowNode, + TRegionReplicaSet dataRegionReplicaSet, + List writePlanNodeList) { + for (int j = 0; j < insertRowNode.getDataTypes().length; j++) { + if (insertRowNode.getDataTypes()[j] == TSDataType.OBJECT) { + Object[] values = insertRowNode.getValues(); + if (values[j] == null) { + continue; + } + byte[] binary = ((Binary) values[j]).getValues(); + ByteBuffer buffer = ByteBuffer.wrap(binary); + boolean isEoF = buffer.get() == 1; + long offset = buffer.getLong(); + byte[] content = ReadWriteIOUtils.readBytes(buffer, buffer.remaining()); + IObjectPath relativePath = + IObjectPath.Factory.FACTORY.create( + dataRegionReplicaSet.getRegionId().getId(), + insertRowNode.getTime(), + insertRowNode.getDeviceID(), + insertRowNode.getMeasurements()[j]); + ObjectNode objectNode = new ObjectNode(isEoF, offset, content, relativePath); + objectNode.setDataRegionReplicaSet(dataRegionReplicaSet); + writePlanNodeList.add(objectNode); + if (isEoF) { + ((Binary) values[j]) + .setValues(generateObjectBinary(offset + content.length, relativePath).getValues()); + insertRowNode.setValues(values); + } else { + values[j] = null; + } + } + } } public RelationalInsertRowsNode emptyClone() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java index 6e5e8eeb5982..3255c11f1e96 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java @@ -30,13 +30,16 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; +import org.apache.iotdb.db.storageengine.dataregion.IObjectPath; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.IWALByteBufferView; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -50,6 +53,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; + +import static org.apache.iotdb.db.utils.ObjectTypeUtils.generateObjectBinary; public class RelationalInsertTabletNode extends InsertTabletNode { @@ -108,6 +114,16 @@ public void setSingleDevice() { this.singleDevice = true; } + public List getObjectColumns() { + List objectColumns = new ArrayList<>(); + for (int i = 0; i < columns.length; i++) { + if (dataTypes[i] == TSDataType.OBJECT) { + objectColumns.add((Binary[]) columns[i]); + } + } + return objectColumns; + } + @Override public IDeviceID getDeviceID(int rowIdx) { if (singleDevice) { @@ -372,4 +388,110 @@ public void updateLastCache(final String databaseName) { startOffset = endOffset; } } + + @Override + protected List doSplit(Map> splitMap) { + List result = new ArrayList<>(); + + if (splitMap.size() == 1) { + final Entry> entry = splitMap.entrySet().iterator().next(); + if (entry.getValue().size() == 2) { + // Avoid using system arraycopy when there is no need to split + setRange(entry.getValue()); + setDataRegionReplicaSet(entry.getKey()); + for (int i = 0; i < columns.length; i++) { + if (dataTypes[i] == TSDataType.OBJECT) { + handleObjectValue(i, 0, times.length, entry, result); + } + } + result.add(this); + return result; + } + } + + for (Map.Entry> entry : splitMap.entrySet()) { + result.addAll(generateOneSplitList(entry)); + } + return result; + } + + private List generateOneSplitList( + Map.Entry> entry) { + List locs; + // generate a new times and values + locs = entry.getValue(); + int count = 0; + for (int i = 0; i < locs.size(); i += 2) { + int start = locs.get(i); + int end = locs.get(i + 1); + count += end - start; + } + + List result = new ArrayList<>(); + + final InsertTabletNode subNode = getEmptySplit(count); + int destLoc = 0; + + for (int k = 0; k < locs.size(); k += 2) { + int start = locs.get(k); + int end = locs.get(k + 1); + final int length = end - start; + + System.arraycopy(times, start, subNode.times, destLoc, length); + for (int i = 0; i < subNode.columns.length; i++) { + if (dataTypes[i] != null) { + if (dataTypes[i] == TSDataType.OBJECT) { + handleObjectValue(i, start, end, entry, result); + } + System.arraycopy(columns[i], start, subNode.columns[i], destLoc, length); + } + if (subNode.bitMaps != null && this.bitMaps[i] != null) { + BitMap.copyOfRange(this.bitMaps[i], start, subNode.bitMaps[i], destLoc, length); + } + } + destLoc += length; + } + subNode.setFailedMeasurementNumber(getFailedMeasurementNumber()); + subNode.setRange(locs); + subNode.setDataRegionReplicaSet(entry.getKey()); + result.add(subNode); + return result; + } + + private void handleObjectValue( + int column, + int startRow, + int endRow, + Map.Entry> entry, + List result) { + for (int j = startRow; j < endRow; j++) { + if (((Binary[]) columns[column])[j] == null) { + continue; + } + byte[] binary = ((Binary[]) columns[column])[j].getValues(); + ByteBuffer buffer = ByteBuffer.wrap(binary); + boolean isEoF = buffer.get() == 1; + long offset = buffer.getLong(); + byte[] content = ReadWriteIOUtils.readBytes(buffer, buffer.remaining()); + IObjectPath relativePath = + IObjectPath.Factory.FACTORY.create( + entry.getKey().getRegionId().getId(), times[j], getDeviceID(j), measurements[column]); + ObjectNode objectNode = new ObjectNode(isEoF, offset, content, relativePath); + objectNode.setDataRegionReplicaSet(entry.getKey()); + result.add(objectNode); + if (isEoF) { + ((Binary[]) columns[column])[j] = + generateObjectBinary(offset + content.length, relativePath); + } else { + ((Binary[]) columns[column])[j] = null; + if (bitMaps == null) { + bitMaps = new BitMap[columns.length]; + } + if (bitMaps[column] == null) { + bitMaps[column] = new BitMap(rowCount); + } + bitMaps[column].mark(j); + } + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index f65ee3689e10..e13c52ba8b16 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -221,6 +221,7 @@ import com.google.common.collect.Streams; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.read.common.type.BinaryType; +import org.apache.tsfile.read.common.type.ObjectType; import org.apache.tsfile.read.common.type.RowType; import org.apache.tsfile.read.common.type.StringType; import org.apache.tsfile.read.common.type.TimestampType; @@ -4777,11 +4778,15 @@ public Scope visitTableFunctionInvocation(TableFunctionInvocation node, Optional i -> i.getFields().stream() .map( - f -> - Field.newUnqualified( - f.getName(), - UDFDataTypeTransformer.transformUDFDataTypeToReadType(f.getType()), - TsTableColumnCategory.FIELD)) + f -> { + Type type = + UDFDataTypeTransformer.transformUDFDataTypeToReadType(f.getType()); + if (type == ObjectType.OBJECT) { + throw new SemanticException( + "OBJECT type is not supported as return type"); + } + return Field.newUnqualified(f.getName(), type, TsTableColumnCategory.FIELD); + }) .forEach(fields::add)); // next, columns derived from table arguments, in order of argument declarations diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index c15071be6978..7786876e19ad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -66,6 +66,7 @@ import org.apache.iotdb.udf.api.relational.TableFunction; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.read.common.type.ObjectType; import org.apache.tsfile.read.common.type.StringType; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.TypeFactory; @@ -266,13 +267,15 @@ && isIntegerNumber(argumentTypes.get(2)))) { return STRING; } else if (TableBuiltinScalarFunction.LENGTH.getFunctionName().equalsIgnoreCase(functionName)) { if (!(argumentTypes.size() == 1 - && (isCharType(argumentTypes.get(0)) || isBlobType(argumentTypes.get(0))))) { + && (isCharType(argumentTypes.get(0)) + || isBlobType(argumentTypes.get(0)) + || isObjectType(argumentTypes.get(0))))) { throw new SemanticException( "Scalar function " + functionName.toLowerCase(Locale.ENGLISH) - + " only accepts one argument and it must be text, string, or blob data type."); + + " only accepts one argument and it must be text or string or blob or object data type."); } - return INT32; + return INT64; } else if (TableBuiltinScalarFunction.UPPER.getFunctionName().equalsIgnoreCase(functionName)) { if (!(argumentTypes.size() == 1 && isCharType(argumentTypes.get(0)))) { throw new SemanticException( @@ -1078,6 +1081,20 @@ && isIntegerNumber(argumentTypes.get(2)))) { functionName)); } return BLOB; + } else if (TableBuiltinScalarFunction.READ_OBJECT + .getFunctionName() + .equalsIgnoreCase(functionName)) { + if (argumentTypes.isEmpty() + || argumentTypes.size() > 3 + || !isObjectType(argumentTypes.get(0)) + || (argumentTypes.size() >= 2 && !isIntegerNumber(argumentTypes.get(1))) + || (argumentTypes.size() >= 3 && !isIntegerNumber(argumentTypes.get(2)))) { + throw new SemanticException( + "Scalar function " + + functionName.toLowerCase(Locale.ENGLISH) + + " must have at 1~3 arguments, and first argument must be OBJECT type, other arguments must be int32 or int64 type"); + } + return BLOB; } // builtin aggregation function @@ -1318,8 +1335,13 @@ && isIntegerNumber(argumentTypes.get(2)))) { Collections.emptyMap()); try { ScalarFunctionAnalysis scalarFunctionAnalysis = scalarFunction.analyze(functionArguments); - return UDFDataTypeTransformer.transformUDFDataTypeToReadType( - scalarFunctionAnalysis.getOutputDataType()); + Type returnType = + UDFDataTypeTransformer.transformUDFDataTypeToReadType( + scalarFunctionAnalysis.getOutputDataType()); + if (returnType == ObjectType.OBJECT) { + throw new SemanticException("OBJECT type is not supported as return type"); + } + return returnType; } catch (Exception e) { throw new SemanticException("Invalid function parameters: " + e.getMessage()); } finally { @@ -1336,8 +1358,13 @@ && isIntegerNumber(argumentTypes.get(2)))) { try { AggregateFunctionAnalysis aggregateFunctionAnalysis = aggregateFunction.analyze(functionArguments); - return UDFDataTypeTransformer.transformUDFDataTypeToReadType( - aggregateFunctionAnalysis.getOutputDataType()); + Type returnType = + UDFDataTypeTransformer.transformUDFDataTypeToReadType( + aggregateFunctionAnalysis.getOutputDataType()); + if (returnType == ObjectType.OBJECT) { + throw new SemanticException("OBJECT type is not supported as return type"); + } + return returnType; } catch (Exception e) { throw new SemanticException("Invalid function parameters: " + e.getMessage()); } finally { @@ -1516,6 +1543,10 @@ public static boolean isCharType(Type type) { return TEXT.equals(type) || StringType.STRING.equals(type); } + public static boolean isObjectType(Type type) { + return ObjectType.OBJECT.equals(type); + } + public static boolean isBlobType(Type type) { return BLOB.equals(type); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java index b50a875d7b08..3be5d9316573 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableDeviceSchemaValidator.java @@ -19,9 +19,9 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher; -import org.apache.iotdb.commons.exception.IoTDBException; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.commons.exception.IoTDBRuntimeException; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.db.exception.sql.SemanticException; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.plan.Coordinator; @@ -53,8 +53,6 @@ public class TableDeviceSchemaValidator { private static final Logger LOGGER = LoggerFactory.getLogger(TableDeviceSchemaValidator.class); - private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private final Coordinator coordinator = Coordinator.getInstance(); private final TableDeviceSchemaFetcher fetcher = TableDeviceSchemaFetcher.getInstance(); @@ -247,9 +245,18 @@ private void autoCreateOrUpdateDeviceSchema( Long.MAX_VALUE, false); if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - throw new RuntimeException( - new IoTDBException( - executionResult.status.getMessage(), executionResult.status.getCode())); + throw new IoTDBRuntimeException( + executionResult.status.getMessage(), executionResult.status.getCode()); + } + } + + public static void checkObject4DeviceId(final Object[] deviceId) { + for (final Object part : deviceId) { + final String value = (String) part; + if (Objects.nonNull(value) && TsTable.isInvalid4ObjectType(value)) { + throw new SemanticException( + TsTable.getObjectStringError("deviceId", Arrays.toString(deviceId))); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java index 225137b328ed..9cef48070285 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/AddExchangeNodes.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.read.TableDeviceSourceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.CollectNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExplainAnalyzeNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableFunctionProcessorNode; @@ -68,7 +69,7 @@ public PlanNode visitPlan(PlanNode node, TableDistributedPlanGenerator.PlanConte SAME_WITH_ALL_CHILDREN, context .nodeDistributionMap - .get(node.getChildren().get(0).getPlanNodeId()) + .get(newNode.getChildren().get(0).getPlanNodeId()) .getRegion())); return newNode; } @@ -129,6 +130,9 @@ public PlanNode visitExplainAnalyze( ExchangeNode exchangeNode = new ExchangeNode(queryContext.getQueryId().genPlanNodeId()); exchangeNode.setChild(child); + context.nodeDistributionMap.put( + exchangeNode.getPlanNodeId(), + new NodeDistribution(DIFFERENT_FROM_ALL_CHILDREN, DataPartition.NOT_ASSIGNED)); exchangeNode.setOutputSymbols(child.getOutputSymbols()); newNode.setChild(exchangeNode); @@ -139,6 +143,41 @@ public PlanNode visitExplainAnalyze( return newNode; } + @Override + public PlanNode visitCollect( + CollectNode node, TableDistributedPlanGenerator.PlanContext context) { + PlanNode newNode = node.clone(); + if (node.getChildren().size() == 1) { + newNode.addChild(node.getChildren().get(0).accept(this, context)); + context.nodeDistributionMap.put( + node.getPlanNodeId(), + new NodeDistribution( + SAME_WITH_ALL_CHILDREN, + context + .nodeDistributionMap + .get(newNode.getChildren().get(0).getPlanNodeId()) + .getRegion())); + return newNode; + } + + for (PlanNode child : node.getChildren()) { + PlanNode rewriteNode = child.accept(this, context); + ExchangeNode exchangeNode = new ExchangeNode(queryContext.getQueryId().genPlanNodeId()); + exchangeNode.addChild(rewriteNode); + exchangeNode.setOutputSymbols(rewriteNode.getOutputSymbols()); + newNode.addChild(exchangeNode); + context.hasExchangeNode = true; + context.nodeDistributionMap.put( + exchangeNode.getPlanNodeId(), + new NodeDistribution(DIFFERENT_FROM_ALL_CHILDREN, DataPartition.NOT_ASSIGNED)); + } + context.nodeDistributionMap.put( + newNode.getPlanNodeId(), + new NodeDistribution(DIFFERENT_FROM_ALL_CHILDREN, DataPartition.NOT_ASSIGNED)); + + return newNode; + } + @Override public PlanNode visitTableFunctionProcessor( TableFunctionProcessorNode node, TableDistributedPlanGenerator.PlanContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java index 9b1487b0a48a..ed5bc0d3719e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DeleteDevice.java @@ -226,7 +226,8 @@ public static DeviceBlackListConstructor constructDevicePredicateUpdater( ImmutableList.of(), 0, mockTypeProvider, - metadata)) + metadata, + null)) : null; return new DeviceBlackListConstructor( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java index cbce78e6528c..70b2e0623650 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRow.java @@ -21,11 +21,15 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaValidator; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; +import org.apache.tsfile.enums.TSDataType; + import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; public class InsertRow extends WrappedInsertStatement { @@ -56,7 +60,12 @@ public String getTableName() { @Override public List getDeviceIdList() { final InsertRowStatement insertRowStatement = getInnerTreeStatement(); - Object[] segments = insertRowStatement.getTableDeviceID().getSegments(); + final Object[] segments = insertRowStatement.getTableDeviceID().getSegments(); + if (Objects.nonNull(getInnerTreeStatement().getMeasurementSchemas()) + && Arrays.stream(getInnerTreeStatement().getMeasurementSchemas()) + .anyMatch(schema -> Objects.nonNull(schema) && schema.getType() == TSDataType.OBJECT)) { + TableDeviceSchemaValidator.checkObject4DeviceId(segments); + } return Collections.singletonList(Arrays.copyOfRange(segments, 1, segments.length)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java index d2a0da3c1dd6..d35e68efcf52 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertRows.java @@ -25,13 +25,17 @@ import org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeUtils; import org.apache.iotdb.db.queryengine.plan.relational.metadata.ITableDeviceSchemaValidation; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaValidator; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement; +import org.apache.tsfile.enums.TSDataType; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; public class InsertRows extends WrappedInsertStatement { @@ -117,8 +121,14 @@ public String getTableName() { @Override public List getDeviceIdList() { - Object[] idSegments = insertRowStatement.getTableDeviceID().getSegments(); - return Collections.singletonList(Arrays.copyOfRange(idSegments, 1, idSegments.length)); + final Object[] tagSegments = insertRowStatement.getTableDeviceID().getSegments(); + if (Objects.nonNull(insertRowStatement.getMeasurementSchemas()) + && Arrays.stream(insertRowStatement.getMeasurementSchemas()) + .anyMatch( + schema -> Objects.nonNull(schema) && schema.getType() == TSDataType.OBJECT)) { + TableDeviceSchemaValidator.checkObject4DeviceId(tagSegments); + } + return Collections.singletonList(Arrays.copyOfRange(tagSegments, 1, tagSegments.length)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java index 9894adb4d01b..8c1bccf28aba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/InsertTablet.java @@ -21,8 +21,10 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaValidator; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import java.util.ArrayList; @@ -30,6 +32,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; public class InsertTablet extends WrappedInsertStatement { @@ -65,6 +68,12 @@ public List getDeviceIdList() { List deviceIdList = new ArrayList<>(); for (IDeviceID deviceID : deviceID2LastIdxMap.keySet()) { Object[] segments = deviceID.getSegments(); + if (Objects.nonNull(super.getInnerTreeStatement().getMeasurementSchemas()) + && Arrays.stream(super.getInnerTreeStatement().getMeasurementSchemas()) + .anyMatch( + schema -> Objects.nonNull(schema) && schema.getType() == TSDataType.OBJECT)) { + TableDeviceSchemaValidator.checkObject4DeviceId(segments); + } deviceIdList.add(Arrays.copyOfRange(segments, 1, segments.length)); } return deviceIdList; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/AstUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/AstUtil.java index cb17a90c04f3..700d5e40beb8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/AstUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/AstUtil.java @@ -21,16 +21,22 @@ import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BinaryLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TableExpressionType; import com.google.common.graph.SuccessorsFunction; import com.google.common.graph.Traverser; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; import java.time.ZoneId; import java.util.List; @@ -113,6 +119,30 @@ public static Object expressionToTsValue(Expression expression) { throw new SemanticException( String.format("Cannot insert identifier %s, please use string literal", expression)); } + if (expression instanceof FunctionCall + && "to_object".equals(((FunctionCall) expression).getName().toString())) { + List arguments = ((FunctionCall) expression).getArguments(); + if (arguments.size() == 3 + && arguments.get(0).getExpressionType() == TableExpressionType.BOOLEAN_LITERAL + && arguments.get(1).getExpressionType() == TableExpressionType.LONG_LITERAL + && arguments.get(2).getExpressionType() == TableExpressionType.BINARY_LITERAL) { + boolean isEOF = + (boolean) + ((BooleanLiteral) ((FunctionCall) expression).getArguments().get(0)).getTsValue(); + long offset = + (long) ((LongLiteral) ((FunctionCall) expression).getArguments().get(1)).getTsValue(); + byte[] content = + ((Binary) + ((BinaryLiteral) ((FunctionCall) expression).getArguments().get(2)) + .getTsValue()) + .getValues(); + byte[] val = new byte[content.length + 9]; + val[0] = (byte) (isEOF ? 1 : 0); + System.arraycopy(BytesUtils.longToBytes(offset), 0, val, 1, 8); + System.arraycopy(content, 0, val, 9, content.length); + return new Binary(val); + } + } throw new SemanticException("Unsupported expression: " + expression); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/CompatibleResolver.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/CompatibleResolver.java index a33f93692db5..2f71195203f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/CompatibleResolver.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/CompatibleResolver.java @@ -34,6 +34,7 @@ import static org.apache.tsfile.read.common.type.FloatType.FLOAT; import static org.apache.tsfile.read.common.type.IntType.INT32; import static org.apache.tsfile.read.common.type.LongType.INT64; +import static org.apache.tsfile.read.common.type.ObjectType.OBJECT; import static org.apache.tsfile.read.common.type.StringType.STRING; import static org.apache.tsfile.read.common.type.TimestampType.TIMESTAMP; import static org.apache.tsfile.read.common.type.UnknownType.UNKNOWN; @@ -89,6 +90,9 @@ public class CompatibleResolver { addCondition(BLOB, BLOB, BLOB); addCondition(BLOB, UNKNOWN, BLOB); + addCondition(OBJECT, OBJECT, OBJECT); + addCondition(OBJECT, UNKNOWN, OBJECT); + addCondition(UNKNOWN, INT32, INT32); addCondition(UNKNOWN, INT64, INT64); addCondition(UNKNOWN, FLOAT, FLOAT); @@ -99,6 +103,7 @@ public class CompatibleResolver { addCondition(UNKNOWN, TEXT, TEXT); addCondition(UNKNOWN, STRING, STRING); addCondition(UNKNOWN, BLOB, BLOB); + addCondition(UNKNOWN, OBJECT, OBJECT); addCondition(UNKNOWN, UNKNOWN, UNKNOWN); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java index 6f2aaef76ab1..5719a4555ff1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java @@ -35,6 +35,7 @@ import static org.apache.tsfile.read.common.type.FloatType.FLOAT; import static org.apache.tsfile.read.common.type.IntType.INT32; import static org.apache.tsfile.read.common.type.LongType.INT64; +import static org.apache.tsfile.read.common.type.ObjectType.OBJECT; import static org.apache.tsfile.read.common.type.StringType.STRING; import static org.apache.tsfile.read.common.type.TimestampType.TIMESTAMP; import static org.apache.tsfile.read.common.type.UnknownType.UNKNOWN; @@ -52,6 +53,7 @@ public InternalTypeManager() { types.put(new TypeSignature(TypeEnum.TEXT.name().toLowerCase(Locale.ENGLISH)), TEXT); types.put(new TypeSignature(TypeEnum.STRING.name().toLowerCase(Locale.ENGLISH)), STRING); types.put(new TypeSignature(TypeEnum.BLOB.name().toLowerCase(Locale.ENGLISH)), BLOB); + types.put(new TypeSignature(TypeEnum.OBJECT.name().toLowerCase(Locale.ENGLISH)), OBJECT); types.put(new TypeSignature(TypeEnum.DATE.name().toLowerCase(Locale.ENGLISH)), DATE); types.put(new TypeSignature(TypeEnum.TIMESTAMP.name().toLowerCase(Locale.ENGLISH)), TIMESTAMP); types.put(new TypeSignature(TypeEnum.UNKNOWN.name().toLowerCase(Locale.ENGLISH)), UNKNOWN); @@ -105,6 +107,8 @@ public static TSDataType getTSDataType(Type type) { return TSDataType.BLOB; case STRING: return TSDataType.STRING; + case OBJECT: + return TSDataType.OBJECT; default: throw new IllegalArgumentException(); } @@ -132,6 +136,8 @@ public static Type fromTSDataType(TSDataType dataType) { return TIMESTAMP; case BLOB: return BLOB; + case OBJECT: + return OBJECT; case STRING: return STRING; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/TypeUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/TypeUtil.java index 0cc31381072f..fe951d1e4ce1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/TypeUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/TypeUtil.java @@ -28,6 +28,7 @@ import org.apache.tsfile.read.common.type.BlobType; import org.apache.tsfile.read.common.type.DoubleType; import org.apache.tsfile.read.common.type.FloatType; +import org.apache.tsfile.read.common.type.ObjectType; import org.apache.tsfile.read.common.type.RowType; import org.apache.tsfile.read.common.type.StringType; import org.apache.tsfile.read.common.type.TimestampType; @@ -116,6 +117,8 @@ public static Type getType(TypeEnum typeEnum, List subTypes) { return TimestampType.TIMESTAMP; case DATE: return DATE; + case OBJECT: + return ObjectType.OBJECT; case ROW: return RowType.anonymous(subTypes); default: @@ -123,7 +126,6 @@ public static Type getType(TypeEnum typeEnum, List subTypes) { } } - // TODO move these methods into each Type to avoid branch miss public static boolean isFlatVariableWidth(Type type) { switch (type.getTypeEnum()) { case BOOLEAN: @@ -137,6 +139,7 @@ public static boolean isFlatVariableWidth(Type type) { case TEXT: case STRING: case BLOB: + case OBJECT: return true; default: throw new UnsupportedOperationException(); @@ -160,6 +163,7 @@ public static int getFlatFixedSize(Type type) { case TEXT: case STRING: case BLOB: + case OBJECT: return 16; default: throw new UnsupportedOperationException(); @@ -179,6 +183,7 @@ public static int getFlatVariableWidthSize(Type type, Column column, int positio case TEXT: case STRING: case BLOB: + case OBJECT: return column.isNull(position) ? 0 : column.getBinary(position).getLength(); default: throw new UnsupportedOperationException(); @@ -198,6 +203,7 @@ public static int getFlatVariableWidthSize(Type type, Column column, int[] posit case TEXT: case STRING: case BLOB: + case OBJECT: int result = 0; for (int i = 0; i < position.length; i++) { if (!column.isNull(i)) { @@ -237,6 +243,7 @@ public static void readFlat( case TEXT: case STRING: case BLOB: + case OBJECT: int length = bytesToInt(fixedChunk, fixedOffset); byte[] result = new byte[length]; if (length <= 12) { @@ -282,6 +289,7 @@ public static void writeFlat( case TEXT: case STRING: case BLOB: + case OBJECT: byte[] value = column.getBinary(position).getValues(); intToBytes(value.length, fixedChunk, fixedOffset); if (value.length <= 12) { @@ -320,6 +328,7 @@ public static boolean notDistinctFrom( case TEXT: case STRING: case BLOB: + case OBJECT: int leftLength = bytesToInt(fixedChunk, fixedOffset); byte[] leftValue = new byte[leftLength]; byte[] rightValue = column.getBinary(position).getValues(); @@ -414,6 +423,7 @@ public static long hash(Type type, byte[] fixedChunk, int fixedOffset, byte[] va case TEXT: case STRING: case BLOB: + case OBJECT: int length = bytesToInt(fixedChunk, fixedOffset); byte[] values = new byte[length]; @@ -490,6 +500,7 @@ public static long hash(Type type, Column column, int position) { case TEXT: case STRING: case BLOB: + case OBJECT: return XxHash64.hash(column.getBinary(position).getValues()); default: throw new UnsupportedOperationException(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java index 7e61bea8a4a8..18ab1cdc2754 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java @@ -328,7 +328,7 @@ private Future dispatchWrite(List if (dispatchFailures.isEmpty()) { return immediateFuture(new FragInstanceDispatchResult(true)); } - if (instances.size() == 1) { + if (instances.size() == 1 || dispatchFailures.size() == 1) { return immediateFuture(new FragInstanceDispatchResult(dispatchFailures.get(0))); } else { List failureStatusList = new ArrayList<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/udf/UserDefineScalarFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/udf/UserDefineScalarFunctionTransformer.java index 279e3c06fd49..47fd40ed73f9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/udf/UserDefineScalarFunctionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/udf/UserDefineScalarFunctionTransformer.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.queryengine.transformation.dag.column.udf; -import org.apache.iotdb.commons.udf.access.RecordIterator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.RecordIterator; import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.multi.MultiColumnTransformer; import org.apache.iotdb.udf.api.relational.ScalarFunction; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/AbstractCastFunctionColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/AbstractCastFunctionColumnTransformer.java index ee40a3ec6f8e..9ed1bb33075d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/AbstractCastFunctionColumnTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/AbstractCastFunctionColumnTransformer.java @@ -341,6 +341,7 @@ protected void castString(ColumnBuilder columnBuilder, Binary value) { case TEXT: case STRING: case BLOB: + case OBJECT: returnType.writeBinary(columnBuilder, value); break; default: @@ -393,4 +394,13 @@ protected void castBlob(ColumnBuilder columnBuilder, Binary value) { String.format("Cannot cast %s to %s type", stringValue, returnType.getDisplayName())); } } + + protected void castObject(ColumnBuilder columnBuilder, Binary value) { + String stringValue = BytesUtils.parseObjectByteArrayToString(value.getValues()); + if (returnType.getTypeEnum() == TypeEnum.STRING) { + returnType.writeBinary(columnBuilder, BytesUtils.valueOf(String.valueOf(stringValue))); + } else { + throw new UnsupportedOperationException(String.format(ERROR_MSG, returnType.getTypeEnum())); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/AbstractLengthColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/AbstractLengthColumnTransformer.java new file mode 100644 index 000000000000..08eb47691668 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/AbstractLengthColumnTransformer.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.UnaryColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; + +public abstract class AbstractLengthColumnTransformer extends UnaryColumnTransformer { + + public AbstractLengthColumnTransformer( + Type returnType, ColumnTransformer childColumnTransformer) { + super(returnType, childColumnTransformer); + } + + @Override + protected void doTransform(Column column, ColumnBuilder columnBuilder) { + for (int i = 0, n = column.getPositionCount(); i < n; i++) { + if (!column.isNull(i)) { + columnBuilder.writeLong(transformNonNullValue(column.getBinary(i))); + } else { + columnBuilder.appendNull(); + } + } + } + + @Override + protected void doTransform(Column column, ColumnBuilder columnBuilder, boolean[] selection) { + for (int i = 0, n = column.getPositionCount(); i < n; i++) { + if (selection[i] && !column.isNull(i)) { + columnBuilder.writeLong(transformNonNullValue(column.getBinary(i))); + } else { + columnBuilder.appendNull(); + } + } + } + + protected abstract long transformNonNullValue(Binary binary); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformer.java index e94d9ad3907d..bce18fb05971 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformer.java @@ -20,37 +20,18 @@ package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar; import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; -import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.UnaryColumnTransformer; -import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.Binary; -public class BlobLengthColumnTransformer extends UnaryColumnTransformer { +public class BlobLengthColumnTransformer extends AbstractLengthColumnTransformer { public BlobLengthColumnTransformer(Type returnType, ColumnTransformer childColumnTransformer) { super(returnType, childColumnTransformer); } @Override - protected void doTransform(Column column, ColumnBuilder columnBuilder) { - doTransform(column, columnBuilder, null); - } - - @Override - protected void doTransform(Column column, ColumnBuilder columnBuilder, boolean[] selection) { - - int positionCount = column.getPositionCount(); - for (int i = 0; i < positionCount; i++) { - if ((selection != null && !selection[i]) || column.isNull(i)) { - columnBuilder.appendNull(); - continue; - } - - Binary value = column.getBinary(i); - int length = value.getValues().length; - columnBuilder.writeInt(length); - } + protected long transformNonNullValue(Binary binary) { + return binary.getValues().length; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/CastFunctionColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/CastFunctionColumnTransformer.java index b9c8e31b1abb..624eabadc62d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/CastFunctionColumnTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/CastFunctionColumnTransformer.java @@ -67,6 +67,9 @@ protected void transform( case BLOB: castBlob(columnBuilder, childType.getBinary(column, i)); break; + case OBJECT: + castObject(columnBuilder, childType.getBinary(column, i)); + break; default: throw new UnsupportedOperationException( String.format( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/LengthColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/LengthColumnTransformer.java index 00448c6f575b..c94530c83d40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/LengthColumnTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/LengthColumnTransformer.java @@ -20,40 +20,18 @@ package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar; import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; -import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.UnaryColumnTransformer; -import org.apache.tsfile.block.column.Column; -import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; -public class LengthColumnTransformer extends UnaryColumnTransformer { - +public class LengthColumnTransformer extends AbstractLengthColumnTransformer { public LengthColumnTransformer(Type returnType, ColumnTransformer childColumnTransformer) { super(returnType, childColumnTransformer); } @Override - protected void doTransform(Column column, ColumnBuilder columnBuilder) { - for (int i = 0, n = column.getPositionCount(); i < n; i++) { - if (!column.isNull(i)) { - String currentValue = column.getBinary(i).getStringValue(TSFileConfig.STRING_CHARSET); - columnBuilder.writeInt(currentValue.length()); - } else { - columnBuilder.appendNull(); - } - } - } - - @Override - protected void doTransform(Column column, ColumnBuilder columnBuilder, boolean[] selection) { - for (int i = 0, n = column.getPositionCount(); i < n; i++) { - if (selection[i] && !column.isNull(i)) { - String currentValue = column.getBinary(i).getStringValue(TSFileConfig.STRING_CHARSET); - columnBuilder.writeInt(currentValue.length()); - } else { - columnBuilder.appendNull(); - } - } + protected long transformNonNullValue(Binary binary) { + return binary.getStringValue(TSFileConfig.STRING_CHARSET).length(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ObjectLengthColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ObjectLengthColumnTransformer.java new file mode 100644 index 000000000000..5d39c6f6af3c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ObjectLengthColumnTransformer.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.db.utils.ObjectTypeUtils; + +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; + +public class ObjectLengthColumnTransformer extends AbstractLengthColumnTransformer { + public ObjectLengthColumnTransformer(Type returnType, ColumnTransformer childColumnTransformer) { + super(returnType, childColumnTransformer); + } + + @Override + protected long transformNonNullValue(Binary binary) { + return ObjectTypeUtils.getObjectLength(binary); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java new file mode 100644 index 000000000000..a4ad4e25756a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar; + +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.unary.UnaryColumnTransformer; +import org.apache.iotdb.db.utils.ObjectTypeUtils; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.Type; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.Pair; + +import java.util.Optional; + +public class ReadObjectColumnTransformer extends UnaryColumnTransformer { + + private final Optional fragmentInstanceContext; + private long offset = 0; + private long length = -1; + + public ReadObjectColumnTransformer( + Type type, + ColumnTransformer childColumnTransformer, + Optional fragmentInstanceContext) { + super(type, childColumnTransformer); + this.fragmentInstanceContext = fragmentInstanceContext; + } + + public ReadObjectColumnTransformer( + Type type, + long offset, + ColumnTransformer childColumnTransformer, + Optional fragmentInstanceContext) { + super(type, childColumnTransformer); + this.offset = offset; + this.fragmentInstanceContext = fragmentInstanceContext; + } + + public ReadObjectColumnTransformer( + Type type, + long offset, + long length, + ColumnTransformer childColumnTransformer, + Optional fragmentInstanceContext) { + super(type, childColumnTransformer); + this.offset = offset; + this.length = length; + this.fragmentInstanceContext = fragmentInstanceContext; + } + + @Override + protected void doTransform(Column column, ColumnBuilder columnBuilder) { + for (int i = 0, n = column.getPositionCount(); i < n; i++) { + if (!column.isNull(i)) { + transform(column, columnBuilder, i); + } else { + columnBuilder.appendNull(); + } + } + } + + @Override + protected void doTransform(Column column, ColumnBuilder columnBuilder, boolean[] selection) { + for (int i = 0, n = column.getPositionCount(); i < n; i++) { + if (selection[i] && !column.isNull(i)) { + transform(column, columnBuilder, i); + } else { + columnBuilder.appendNull(); + } + } + } + + private void transform(Column column, ColumnBuilder columnBuilder, int i) { + // BinaryColumn.getDataType() returns TSDataType.TEXT + if (TSDataType.TEXT == column.getDataType()) { + Binary binary = column.getBinary(i); + columnBuilder.writeBinary(readObject(binary)); + } + } + + private Binary readObject(Binary binary) { + Pair objectLengthPathPair = + ObjectTypeUtils.parseObjectBinaryToSizeStringPathPair(binary); + long fileLength = objectLengthPathPair.getLeft(); + String relativePath = objectLengthPathPair.getRight(); + int actualReadSize = + ObjectTypeUtils.getActualReadSize(relativePath, fileLength, offset, length); + fragmentInstanceContext.ifPresent( + context -> context.getMemoryReservationContext().reserveMemoryCumulatively(actualReadSize)); + return new Binary( + ObjectTypeUtils.readObjectContent(relativePath, offset, actualReadSize, true).array()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/TryCastFunctionColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/TryCastFunctionColumnTransformer.java index c25fd321c344..419d1bbabf3b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/TryCastFunctionColumnTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/TryCastFunctionColumnTransformer.java @@ -69,6 +69,9 @@ protected void transform( case BLOB: castBlob(columnBuilder, childType.getBinary(column, i)); break; + case OBJECT: + castObject(columnBuilder, childType.getBinary(column, i)); + break; default: throw new UnsupportedOperationException( String.format( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java index 32125c110ef5..14643a7b6ac5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/input/ConstantInputReader.java @@ -92,6 +92,7 @@ public ConstantInputReader(ConstantOperand expression) throws QueryProcessExcept cachedColumns[0] = new RunLengthEncodedColumn(booleanColumn, count); break; case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java index fcbb30b7ca13..59b3dbbc8b63 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/ArithmeticNegationTransformer.java @@ -59,6 +59,7 @@ protected void transform(Column[] columns, ColumnBuilder builder) case TEXT: case TIMESTAMP: case BLOB: + case OBJECT: case BOOLEAN: case STRING: default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java index 5bdfa8f96a7f..09cc6e76ad38 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/InTransformer.java @@ -88,6 +88,7 @@ private void initTypedSet(Set values) { stringSet = values; break; case BLOB: + case OBJECT: default: throw new UnsupportedOperationException("unsupported data type: " + layerReaderDataType); } @@ -124,6 +125,7 @@ protected void transform(Column[] columns, ColumnBuilder builder) transformBinary(columns, builder); return; case BLOB: + case OBJECT: default: throw new QueryProcessException("unsupported data type: " + layerReaderDataType); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java index 11cf952e0b43..195acb6de852 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/CastFunctionTransformer.java @@ -68,6 +68,7 @@ protected Column[] transform(Column[] columns) throws QueryProcessException, IOE case TEXT: return castBinaries(columns); case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case DATE: @@ -139,6 +140,7 @@ private Column[] castInts(Column[] columns) { break; case STRING: case BLOB: + case OBJECT: case TIMESTAMP: case DATE: case INT32: @@ -213,6 +215,7 @@ private Column[] castLongs(Column[] columns) { } break; case BLOB: + case OBJECT: case STRING: case DATE: case TIMESTAMP: @@ -288,6 +291,7 @@ private Column[] castFloats(Column[] columns) { } break; case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case DATE: @@ -363,6 +367,7 @@ private Column[] castDoubles(Column[] columns) { } break; case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case DATE: @@ -439,6 +444,7 @@ private Column[] castBooleans(Column[] columns) { break; case STRING: case BLOB: + case OBJECT: case DATE: case TIMESTAMP: case BOOLEAN: @@ -521,6 +527,7 @@ private Column[] castBinaries(Column[] columns) { case DATE: case STRING: case BLOB: + case OBJECT: case TEXT: default: throw new UnsupportedOperationException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java index ee7ba10e60e4..03081f9206f1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/DiffFunctionTransformer.java @@ -62,6 +62,7 @@ public void transform(Column[] columns, ColumnBuilder builder) throws QueryProce transformDouble(columns, builder); return; case BLOB: + case OBJECT: case TEXT: case DATE: case STRING: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java index a4ddff58bab4..47e426180773 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/transformer/unary/scalar/RoundFunctionTransformer.java @@ -67,6 +67,7 @@ protected void transform(Column[] columns, ColumnBuilder builder) case STRING: case TEXT: case BLOB: + case OBJECT: default: throw new UnsupportedOperationException( String.format("Unsupported source dataType: %s", layerReaderDataType)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java index c05d7adfecc2..015904399c20 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TransformUtils.java @@ -78,6 +78,7 @@ public static Column transformConstantOperandToColumn(ConstantOperand constantOp return new BooleanColumn(1, Optional.empty(), new boolean[] {(boolean) value}); case STRING: case BLOB: + case OBJECT: case DATE: case TIMESTAMP: default: @@ -158,6 +159,7 @@ public static boolean splitWindowForStateWindow( case TIMESTAMP: case DATE: case BLOB: + case OBJECT: case STRING: default: throw new UnsupportedOperationException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java index 0eedb86a31a9..e8c679e2bb55 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/util/TypeUtils.java @@ -50,6 +50,7 @@ public static ColumnBuilder initColumnBuilder(TSDataType type, int count) { case TEXT: case BLOB: case STRING: + case OBJECT: return new BinaryColumnBuilder(null, count); default: throw new UnSupportedDataTypeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java index 6ec6537de362..3bacd1ac5e79 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/row/SerializableRowList.java @@ -107,6 +107,7 @@ protected static int calculateCapacity( case TEXT: case BLOB: case STRING: + case OBJECT: rowLength += MIN_OBJECT_HEADER_SIZE + MIN_ARRAY_HEADER_SIZE + byteArrayLengthForMemoryControl; break; @@ -214,6 +215,7 @@ private Object[] getRowSkipPrefixNulls(int index) { case TEXT: case BLOB: case STRING: + case OBJECT: row[i] = block[i].getBinary(offset); break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java index 5f76cd46c3dc..57ac89f073a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/datastructure/tv/SerializableTVList.java @@ -92,6 +92,7 @@ protected static int calculateCapacity(TSDataType dataType, float memoryLimitInM case TEXT: case STRING: case BLOB: + case OBJECT: rowLength += MIN_OBJECT_HEADER_SIZE + MIN_ARRAY_HEADER_SIZE diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 9b125f4f1114..5f6486f8be4f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1549,7 +1549,8 @@ private DeviceAttributeUpdater constructDevicePredicateUpdater( ImmutableList.of(), 0, mockTypeProvider, - metadata)) + metadata, + null)) : null; final List filterOutputDataTypes = @@ -1576,7 +1577,8 @@ private DeviceAttributeUpdater constructDevicePredicateUpdater( filterOutputDataTypes, inputLocations.size(), mockTypeProvider, - metadata); + metadata, + null); final List attributeNames = updateNode.getAssignments().stream() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index a39b881fe1f4..5a105cf7dd7c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -1548,7 +1548,7 @@ public ISchemaReader getTableDeviceReader(final PartialPath p @Override public ISchemaReader getTableDeviceReader( - String table, List devicePathList) throws MetadataException { + String table, List devicePathList) { throw new UnsupportedOperationException(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java index 8eca2ba39353..67cc75be749a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/FileMetrics.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.service.metrics.file.CompactionFileMetrics; import org.apache.iotdb.db.service.metrics.file.ModsFileMetrics; +import org.apache.iotdb.db.service.metrics.file.ObjectFileMetrics; import org.apache.iotdb.db.service.metrics.file.SystemRelatedFileMetrics; import org.apache.iotdb.db.service.metrics.file.TsFileMetrics; import org.apache.iotdb.db.service.metrics.file.WalFileMetrics; @@ -40,6 +41,7 @@ public class FileMetrics implements IMetricSet { private static final WalFileMetrics WAL_FILE_METRICS = new WalFileMetrics(); private static final SystemRelatedFileMetrics SYSTEM_RELATED_FILE_METRICS = new SystemRelatedFileMetrics(); + private static final ObjectFileMetrics OBJECT_FILE_METRICS = new ObjectFileMetrics(); @Override public void bindTo(AbstractMetricService metricService) { @@ -48,6 +50,7 @@ public void bindTo(AbstractMetricService metricService) { COMPACTION_FILE_METRICS.bindTo(metricService); WAL_FILE_METRICS.bindTo(metricService); SYSTEM_RELATED_FILE_METRICS.bindTo(metricService); + OBJECT_FILE_METRICS.bindTo(metricService); } @Override @@ -57,6 +60,7 @@ public void unbindFrom(AbstractMetricService metricService) { COMPACTION_FILE_METRICS.unbindFrom(metricService); WAL_FILE_METRICS.unbindFrom(metricService); SYSTEM_RELATED_FILE_METRICS.unbindFrom(metricService); + OBJECT_FILE_METRICS.unbindFrom(metricService); } // region TsFile Related Metrics Update @@ -104,6 +108,22 @@ public long getModFileSize() { // endregion + public void increaseObjectFileNum(int num) { + OBJECT_FILE_METRICS.increaseObjectFileNum(num); + } + + public void decreaseObjectFileNum(int num) { + OBJECT_FILE_METRICS.decreaseObjectFileNum(num); + } + + public void increaseObjectFileSize(long size) { + OBJECT_FILE_METRICS.increaseObjectFileSize(size); + } + + public void decreaseObjectFileSize(long size) { + OBJECT_FILE_METRICS.decreaseObjectFileSize(size); + } + public Map getRegionSizeMap() { return TS_FILE_METRICS.getRegionSizeMap(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ObjectFileMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ObjectFileMetrics.java new file mode 100644 index 000000000000..71e68c235df8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/file/ObjectFileMetrics.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics.file; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class ObjectFileMetrics implements IMetricSet { + private static final String OBJECT = "object"; + private final AtomicInteger objFileNum = new AtomicInteger(0); + private final AtomicLong objFileSize = new AtomicLong(0); + + @Override + public void bindTo(AbstractMetricService metricService) { + metricService.createAutoGauge( + Metric.FILE_SIZE.toString(), + MetricLevel.CORE, + this, + ObjectFileMetrics::getObjectFileSize, + Tag.NAME.toString(), + OBJECT); + metricService.createAutoGauge( + Metric.FILE_COUNT.toString(), + MetricLevel.CORE, + this, + ObjectFileMetrics::getObjectFileNum, + Tag.NAME.toString(), + OBJECT); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + metricService.remove( + MetricType.AUTO_GAUGE, Metric.FILE_SIZE.toString(), Tag.NAME.toString(), OBJECT); + metricService.remove( + MetricType.AUTO_GAUGE, Metric.FILE_COUNT.toString(), Tag.NAME.toString(), OBJECT); + } + + public int getObjectFileNum() { + return objFileNum.get(); + } + + public long getObjectFileSize() { + return objFileSize.get(); + } + + public void increaseObjectFileNum(int num) { + objFileNum.addAndGet(num); + } + + public void decreaseObjectFileNum(int num) { + objFileNum.addAndGet(-num); + } + + public void increaseObjectFileSize(long size) { + objFileSize.addAndGet(size); + } + + public void decreaseObjectFileSize(long size) { + objFileSize.addAndGet(-size); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index 8c3ac35c5d25..43e76e5a2b2d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -76,6 +76,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.recover.WALRecoverManager; import org.apache.iotdb.db.storageengine.load.LoadTsFileManager; import org.apache.iotdb.db.storageengine.load.limiter.LoadTsFileRateLimiter; +import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; import org.apache.iotdb.db.utils.ThreadUtils; import org.apache.iotdb.rpc.RpcUtils; @@ -83,6 +84,8 @@ import org.apache.tsfile.exception.write.PageException; import org.apache.tsfile.external.commons.io.FileUtils; +import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.fileSystem.fsFactory.FSFactory; import org.apache.tsfile.utils.FilePathUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -90,6 +93,8 @@ import java.io.File; import java.io.IOException; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -106,6 +111,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -119,6 +125,8 @@ public class StorageEngine implements IService { private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); private static final WritingMetrics WRITING_METRICS = WritingMetrics.getInstance(); + private final FSFactory fsFactory = FSFactoryProducer.getFSFactory(); + /** * a folder (system/databases/ by default) that persist system info. Each database will have a * subfolder under the systemDir. @@ -156,6 +164,8 @@ public class StorageEngine implements IService { private final LoadTsFileManager loadTsFileManager = new LoadTsFileManager(); + public final AtomicLong objectFileId = new AtomicLong(0); + private StorageEngine() {} public static StorageEngine getInstance() { @@ -226,6 +236,8 @@ private void asyncRecoverDataRegion() throws StartupException { } private void asyncRecover(List> futures) { + checkObjectFiles(); + Map> localDataRegionInfo = getLocalDataRegionInfo(); localDataRegionInfo.values().forEach(list -> recoverDataRegionNum += list.size()); readyDataRegionNum = new AtomicInteger(0); @@ -1066,6 +1078,38 @@ public static File getDataRegionSystemDir(String dataBaseName, String dataRegion systemDir + File.separator + dataBaseName, dataRegionId); } + private void checkObjectFiles() { + List folders = TierManager.getInstance().getAllObjectFileFolders(); + for (String baseDir : folders) { + File fileFolder = fsFactory.getFile(baseDir); + try (Stream paths = Files.walk(fileFolder.toPath())) { + paths + .filter(Files::isRegularFile) + .filter( + path -> { + String name = path.getFileName().toString(); + return name.endsWith(".bin.back") || name.endsWith(".bin"); + }) + .forEach( + path -> { + String name = path.getFileName().toString(); + if (name.endsWith(".bin.back")) { + try { + Files.delete(path); + } catch (IOException e) { + LOGGER.error("Failed to delete: {} -> {}", path, e.getMessage()); + } + } else if (name.endsWith(".bin")) { + FileMetrics.getInstance().increaseObjectFileNum(1); + FileMetrics.getInstance().increaseObjectFileSize(path.toFile().length()); + } + }); + } catch (IOException e) { + LOGGER.error("Failed to check Object Files: {}", e.getMessage()); + } + } + } + public Runnable executeCompactFileTimeIndexCache() { return () -> { if (!isReadyForNonReadWriteFunctions()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/Base32ObjectPath.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/Base32ObjectPath.java new file mode 100644 index 000000000000..d0ea395502a9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/Base32ObjectPath.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion; + +import com.google.common.io.BaseEncoding; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.ReadWriteForEncodingUtils; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class Base32ObjectPath implements IObjectPath { + + private final Path path; + private int serializedSize = -1; + + private static final Deserializer DESERIALIZER = + new Deserializer() { + @Override + public IObjectPath deserializeFrom(ByteBuffer byteBuffer) { + return deserialize(byteBuffer); + } + + @Override + public IObjectPath deserializeFrom(InputStream inputStream) throws IOException { + return deserialize(inputStream); + } + + @Override + public IObjectPath deserializeFromObjectValue(ByteBuffer byteBuffer) { + return deserialize(byteBuffer); + } + }; + + private static final Factory FACTORY = Base32ObjectPath::new; + + private Base32ObjectPath(String first, String... more) { + path = Paths.get(first, more); + } + + public Base32ObjectPath(Path path) { + this.path = path; + } + + public Base32ObjectPath(int regionId, long time, IDeviceID iDeviceID, String measurement) { + Object[] segments = iDeviceID.getSegments(); + String[] pathSegments = new String[segments.length + 2]; + for (int i = 0; i < segments.length; i++) { + Object segment = segments[i]; + String segmentString = segment == null ? "null" : segment.toString(); + pathSegments[i] = + BaseEncoding.base32() + .omitPadding() + .encode(segmentString.getBytes(StandardCharsets.UTF_8)); + } + pathSegments[pathSegments.length - 2] = + BaseEncoding.base32().omitPadding().encode(measurement.getBytes(StandardCharsets.UTF_8)); + pathSegments[pathSegments.length - 1] = time + ".bin"; + path = Paths.get(String.valueOf(regionId), pathSegments); + } + + @Override + public int serialize(ByteBuffer byteBuffer) { + int cnt = 0; + cnt += ReadWriteForEncodingUtils.writeUnsignedVarInt(path.getNameCount(), byteBuffer); + for (Path segment : path) { + cnt += ReadWriteIOUtils.writeVar(segment.toString(), byteBuffer); + } + return cnt; + } + + @Override + public int serialize(OutputStream outputStream) throws IOException { + int cnt = 0; + cnt += ReadWriteForEncodingUtils.writeUnsignedVarInt(path.getNameCount(), outputStream); + for (Path segment : path) { + cnt += ReadWriteIOUtils.writeVar(segment.toString(), outputStream); + } + return cnt; + } + + @Override + public int getSerializedSize() { + if (serializedSize != -1) { + return serializedSize; + } + int cnt = ReadWriteForEncodingUtils.varIntSize(path.getNameCount()); + for (Path segment : path) { + byte[] bytes = segment.toString().getBytes(StandardCharsets.UTF_8); + cnt += ReadWriteForEncodingUtils.varIntSize(bytes.length); + cnt += bytes.length; + } + serializedSize = cnt; + return cnt; + } + + @Override + public void serializeToObjectValue(ByteBuffer byteBuffer) { + serialize(byteBuffer); + } + + @Override + public int getSerializeSizeToObjectValue() { + return getSerializedSize(); + } + + public static Base32ObjectPath deserialize(ByteBuffer byteBuffer) { + int cnt = ReadWriteForEncodingUtils.readUnsignedVarInt(byteBuffer); + String first = ReadWriteIOUtils.readVarIntString(byteBuffer); + String[] more = new String[cnt - 1]; + + for (int i = 0; i < cnt - 1; ++i) { + more[i] = ReadWriteIOUtils.readVarIntString(byteBuffer); + } + return new Base32ObjectPath(first, more); + } + + public static Base32ObjectPath deserialize(InputStream stream) throws IOException { + int cnt = ReadWriteForEncodingUtils.readUnsignedVarInt(stream); + String first = ReadWriteIOUtils.readVarIntString(stream); + String[] more = new String[cnt - 1]; + + for (int i = 0; i < cnt - 1; ++i) { + more[i] = ReadWriteIOUtils.readVarIntString(stream); + } + + return new Base32ObjectPath(first, more); + } + + @Override + public String toString() { + return path.toString(); + } + + public Path getPath() { + return path; + } + + public static Factory getFACTORY() { + return FACTORY; + } + + public static Deserializer getDESERIALIZER() { + return DESERIALIZER; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 9293f9867b67..09c9ee843041 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -84,6 +84,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsOfOneDeviceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.LastCacheLoadStrategy; @@ -157,6 +158,7 @@ import org.apache.iotdb.db.utils.DateTimeUtils; import org.apache.iotdb.db.utils.EncryptDBUtils; import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.iotdb.db.utils.ObjectWriter; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -185,7 +187,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -1338,7 +1342,10 @@ private boolean executeInsertTablet( InsertTabletNode insertTabletNode, TSStatus[] results, long[] infoForMetrics) throws OutOfTTLException { boolean noFailure; - int loc = insertTabletNode.checkTTL(results, getTTL(insertTabletNode)); + int loc = + insertTabletNode.shouldCheckTTL() + ? insertTabletNode.checkTTL(results, getTTL(insertTabletNode)) + : 0; noFailure = loc == 0; List> deviceEndOffsetPairs = insertTabletNode.splitByDevice(loc, insertTabletNode.getRowCount()); @@ -1946,6 +1953,7 @@ public void syncDeleteDataFiles() throws TsFileProcessorException { } }); deleteAllSGFolders(TierManager.getInstance().getAllFilesFolders()); + deleteAllObjectFiles(TierManager.getInstance().getAllObjectFileFolders()); this.workSequenceTsFileProcessors.clear(); this.workUnsequenceTsFileProcessors.clear(); this.tsFileManager.clear(); @@ -1983,6 +1991,39 @@ private void deleteAllSGFolders(List folder) { } } + private void deleteAllObjectFiles(List folders) { + for (String objectFolder : folders) { + File dataRegionObjectFolder = fsFactory.getFile(objectFolder, dataRegionIdString); + try (Stream paths = Files.walk(dataRegionObjectFolder.toPath())) { + paths + .filter(Files::isRegularFile) + .filter( + path -> { + String name = path.getFileName().toString(); + return name.endsWith(".bin"); + }) + .forEach( + path -> { + FileMetrics.getInstance().decreaseObjectFileNum(1); + FileMetrics.getInstance().decreaseObjectFileSize(path.toFile().length()); + }); + } catch (IOException e) { + logger.error("Failed to check Object Files: {}", e.getMessage()); + } + if (FSUtils.getFSType(dataRegionObjectFolder) != FSType.LOCAL) { + try { + fsFactory.deleteDirectory(dataRegionObjectFolder.getPath()); + } catch (IOException e) { + logger.error("Fail to delete data region object folder {}", dataRegionObjectFolder); + } + } else { + if (dataRegionObjectFolder.exists()) { + org.apache.iotdb.commons.utils.FileUtils.deleteFileOrDirectory(dataRegionObjectFolder); + } + } + } + } + public void timedFlushSeqMemTable() { int count = 0; writeLock("timedFlushSeqMemTable"); @@ -3540,6 +3581,47 @@ public int compact() { } } + public void writeObject(ObjectNode objectNode) throws Exception { + writeLock("writeObject"); + try { + String relativeTmpPathString = objectNode.getFilePathString() + ".tmp"; + String objectFileDir = TierManager.getInstance().getNextFolderForObjectFile(); + File objectTmpFile = + FSFactoryProducer.getFSFactory().getFile(objectFileDir, relativeTmpPathString); + try (ObjectWriter writer = new ObjectWriter(objectTmpFile)) { + writer.write( + objectNode.isGeneratedByRemoteConsensusLeader(), + objectNode.getOffset(), + objectNode.getContent()); + } + if (objectNode.isEOF()) { + File objectFile = + FSFactoryProducer.getFSFactory().getFile(objectFileDir, objectNode.getFilePathString()); + if (objectFile.exists()) { + String relativeBackPathString = objectNode.getFilePathString() + ".back"; + File objectBackFile = + FSFactoryProducer.getFSFactory().getFile(objectFileDir, relativeBackPathString); + Files.move( + objectFile.toPath(), objectBackFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + Files.move( + objectTmpFile.toPath(), objectFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + FileMetrics.getInstance().decreaseObjectFileNum(1); + FileMetrics.getInstance().decreaseObjectFileSize(objectBackFile.length()); + Files.delete(objectBackFile.toPath()); + } else { + Files.move( + objectTmpFile.toPath(), objectFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } + FileMetrics.getInstance().increaseObjectFileNum(1); + FileMetrics.getInstance().increaseObjectFileSize(objectFile.length()); + } + getWALNode() + .ifPresent(walNode -> walNode.log(TsFileProcessor.MEMTABLE_NOT_EXIST, objectNode)); + } finally { + writeUnlock(); + } + } + /** * Load a new tsfile to unsequence dir. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IObjectPath.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IObjectPath.java new file mode 100644 index 000000000000..c340aae440c0 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IObjectPath.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; + +import org.apache.tsfile.file.metadata.IDeviceID; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +public interface IObjectPath { + + IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig(); + + int serialize(ByteBuffer byteBuffer); + + int serialize(OutputStream outputStream) throws IOException; + + int getSerializedSize(); + + void serializeToObjectValue(ByteBuffer byteBuffer); + + int getSerializeSizeToObjectValue(); + + interface Factory { + + IObjectPath create(int regionId, long time, IDeviceID iDeviceID, String measurement); + + Factory FACTORY = + CONFIG.getRestrictObjectLimit() + ? PlainObjectPath.getFACTORY() + : Base32ObjectPath.getFACTORY(); + } + + interface Deserializer { + + IObjectPath deserializeFrom(ByteBuffer byteBuffer); + + IObjectPath deserializeFrom(InputStream inputStream) throws IOException; + + IObjectPath deserializeFromObjectValue(ByteBuffer byteBuffer); + } + + static Deserializer getDeserializer() { + return CONFIG.getRestrictObjectLimit() + ? PlainObjectPath.getDESERIALIZER() + : Base32ObjectPath.getDESERIALIZER(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PlainObjectPath.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PlainObjectPath.java new file mode 100644 index 000000000000..3ae3a925a3c9 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/PlainObjectPath.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion; + +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; + +public class PlainObjectPath implements IObjectPath { + + private final String filePath; + + private static final Deserializer DESERIALIZER = + new Deserializer() { + @Override + public IObjectPath deserializeFrom(ByteBuffer byteBuffer) { + return deserialize(byteBuffer); + } + + @Override + public IObjectPath deserializeFrom(InputStream inputStream) throws IOException { + return deserialize(inputStream); + } + + @Override + public IObjectPath deserializeFromObjectValue(ByteBuffer byteBuffer) { + return deserializeObjectValue(byteBuffer); + } + }; + + private static final Factory FACTORY = PlainObjectPath::new; + + public PlainObjectPath(String filePath) { + this.filePath = filePath; + } + + public PlainObjectPath(int regionId, long time, IDeviceID iDeviceID, String measurement) { + String objectFileName = time + ".bin"; + Object[] segments = iDeviceID.getSegments(); + StringBuilder relativePathString = + new StringBuilder(String.valueOf(regionId)).append(File.separator); + for (Object segment : segments) { + relativePathString + .append(segment == null ? "null" : segment.toString().toLowerCase()) + .append(File.separator); + } + relativePathString.append(measurement).append(File.separator); + relativePathString.append(objectFileName); + this.filePath = relativePathString.toString(); + } + + @Override + public int serialize(ByteBuffer byteBuffer) { + return ReadWriteIOUtils.write(filePath, byteBuffer); + } + + @Override + public int serialize(OutputStream outputStream) throws IOException { + return ReadWriteIOUtils.write(filePath, outputStream); + } + + @Override + public int getSerializedSize() { + return ReadWriteIOUtils.sizeToWrite(filePath); + } + + @Override + public void serializeToObjectValue(ByteBuffer byteBuffer) { + byteBuffer.put(filePath.getBytes(StandardCharsets.UTF_8)); + } + + @Override + public int getSerializeSizeToObjectValue() { + return filePath.getBytes(StandardCharsets.UTF_8).length; + } + + public static PlainObjectPath deserialize(ByteBuffer byteBuffer) { + String filePath = ReadWriteIOUtils.readString(byteBuffer); + return new PlainObjectPath(filePath); + } + + public static PlainObjectPath deserialize(InputStream stream) throws IOException { + String filePath = ReadWriteIOUtils.readString(stream); + return new PlainObjectPath(filePath); + } + + public static PlainObjectPath deserializeObjectValue(ByteBuffer byteBuffer) { + return new PlainObjectPath(StandardCharsets.UTF_8.decode(byteBuffer).toString()); + } + + @Override + public String toString() { + return filePath; + } + + public static Factory getFACTORY() { + return FACTORY; + } + + public static Deserializer getDESERIALIZER() { + return DESERIALIZER; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java index 91184aaec82c..3a5695a73763 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java @@ -181,8 +181,7 @@ public void perform() throws Exception { sortedSourceFiles.addAll(unseqFiles); boolean isTreeModel = !isAligned || device.getTableName().startsWith("root."); long ttl = deviceIterator.getTTLForCurrentDevice(); - sortedSourceFiles.removeIf( - x -> x.definitelyNotContains(device) || !x.isDeviceAlive(device, ttl)); + sortedSourceFiles.removeIf(x -> x.definitelyNotContains(device)); // checked above //noinspection OptionalGetWithoutIsPresent sortedSourceFiles.sort(Comparator.comparingLong(x -> x.getStartTime(device).get())); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java index 41d859924f07..290466ffeb6b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionRecoverException; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICrossCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogAnalyzer; @@ -192,6 +193,11 @@ public boolean doCompaction() { performer.setTargetFiles(targetTsfileResourceList); performer.setSummary(summary); performer.perform(); + if (performer instanceof ReadPointCompactionPerformer) { + for (TsFileResource resource : getAllSourceTsFiles()) { + CompactionUtils.removeDeletedObjectFiles(resource); + } + } CompactionUtils.updateProgressIndexAndMark( targetTsfileResourceList, selectedSequenceFiles, selectedUnsequenceFiles); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java index c61d2275ac1a..034e4eb48b8f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java @@ -30,6 +30,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.IInnerCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogAnalyzer; @@ -389,6 +390,11 @@ protected void compact(SimpleCompactionLogger compactionLogger) throws Exception performer.setTargetFiles(filesView.targetFilesInPerformer); performer.setSummary(summary); performer.perform(); + if (performer instanceof ReadPointCompactionPerformer) { + for (TsFileResource resource : filesView.sourceFilesInCompactionPerformer) { + CompactionUtils.removeDeletedObjectFiles(resource); + } + } prepareTargetFiles(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java index 0967089ecb31..7543d56ce80d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task; +import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.db.service.metrics.FileMetrics; import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionRecoverException; @@ -209,7 +210,7 @@ protected boolean doCompaction() { return isSuccess; } - public boolean settleWithFullyDirtyFiles() { + public boolean settleWithFullyDirtyFiles() throws IllegalPathException, IOException { if (fullyDirtyFiles.isEmpty()) { return true; } @@ -218,6 +219,8 @@ public boolean settleWithFullyDirtyFiles() { if (recoverMemoryStatus) { tsFileManager.remove(resource, resource.isSeq()); } + CompactionUtils.removeDeletedObjectFiles(resource); + boolean res = deleteTsFileOnDisk(resource); if (res) { fullyDeletedSuccessNum++; @@ -293,7 +296,7 @@ public void recover() { } } - public void recoverFullyDirtyFiles() { + public void recoverFullyDirtyFiles() throws IllegalPathException, IOException { if (!settleWithFullyDirtyFiles()) { throw new CompactionRecoverException("Failed to delete fully_dirty source file."); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java index 4f58e8fc1323..bac487291671 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java @@ -31,6 +31,8 @@ import org.apache.iotdb.db.service.metrics.CompactionMetrics; import org.apache.iotdb.db.service.metrics.FileMetrics; import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.reader.CompactionAlignedChunkReader; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.reader.CompactionChunkReader; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.FullExactMatch; @@ -42,21 +44,33 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.db.utils.ModificationUtils; +import org.apache.iotdb.db.utils.ObjectTypeUtils; import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.metrics.utils.SystemMetric; import org.apache.tsfile.common.constant.TsFileConstant; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.header.PageHeader; +import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; +import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.Chunk; import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.read.reader.IPointReader; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.write.writer.TsFileIOWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -497,4 +511,174 @@ public static ModEntry convertTtlToDeletion(IDeviceID deviceID, long timeLowerBo new TimeRange(Long.MIN_VALUE, timeLowerBound)); } } + + public static void removeDeletedObjectFiles(TsFileResource resource) { + // check for compaction recovery + if (!resource.tsFileExists()) { + return; + } + try (MultiTsFileDeviceIterator deviceIterator = + new MultiTsFileDeviceIterator(Collections.singletonList(resource))) { + while (deviceIterator.hasNextDevice()) { + deviceIterator.nextDevice(); + deviceIterator.getReaderAndChunkMetadataForCurrentAlignedSeries(); + } + } catch (Exception e) { + logger.warn("Failed to remove object files from file {}", resource.getTsFilePath(), e); + } + } + + @SuppressWarnings("java:S3776") + public static void removeDeletedObjectFiles( + TsFileSequenceReader reader, + List alignedChunkMetadataList, + List timeMods, + List> valueMods, + int currentRegionId) + throws IOException { + if (alignedChunkMetadataList.isEmpty()) { + return; + } + List objectColumnIndexList = new ArrayList<>(); + List> objectDeletionIntervalList = new ArrayList<>(); + boolean objectColumnHasDeletion = false; + + TSDataType[] dataTypes = new TSDataType[valueMods.size()]; + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + boolean hasNull = false; + for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) { + if (dataTypes[i] != null) { + continue; + } + IChunkMetadata chunkMetadata = alignedChunkMetadata.getValueChunkMetadataList().get(i); + if (chunkMetadata == null) { + hasNull = true; + continue; + } + dataTypes[i] = chunkMetadata.getDataType(); + if (dataTypes[i] == TSDataType.OBJECT) { + objectColumnIndexList.add(i); + List deletionInterval = ModificationUtils.sortAndMerge(valueMods.get(i)); + objectColumnHasDeletion |= (!deletionInterval.isEmpty() || !timeMods.isEmpty()); + objectDeletionIntervalList.add(deletionInterval); + } + } + if (!hasNull) { + break; + } + } + if (!objectColumnHasDeletion) { + return; + } + int[] deletionCursors = new int[objectColumnIndexList.size() + 1]; + List timeDeletionIntervalList = ModificationUtils.sortAndMerge(timeMods); + for (AbstractAlignedChunkMetadata alignedChunkMetadata : alignedChunkMetadataList) { + CompactionUtils.removeDeletedObjectFiles( + reader, + alignedChunkMetadata, + objectColumnIndexList, + timeDeletionIntervalList, + objectDeletionIntervalList, + deletionCursors, + currentRegionId); + } + } + + @SuppressWarnings("java:S3776") + private static void removeDeletedObjectFiles( + TsFileSequenceReader reader, + AbstractAlignedChunkMetadata alignedChunkMetadata, + List objectColumnIndexList, + List timeDeletions, + List> objectDeletions, + int[] deletionCursors, + int currentRegionId) + throws IOException { + Chunk timeChunk = + reader.readMemChunk((ChunkMetadata) alignedChunkMetadata.getTimeChunkMetadata()); + CompactionChunkReader compactionChunkReader = new CompactionChunkReader(timeChunk); + List> timePages = + compactionChunkReader.readPageDataWithoutUncompressing(); + + List valueChunks = new ArrayList<>(); + List>> valuePages = new ArrayList<>(); + + for (int i = 0; i < objectColumnIndexList.size(); i++) { + int idxInAlignedChunkMetadata = objectColumnIndexList.get(i); + if (timeDeletions.isEmpty() && objectDeletions.get(i).isEmpty()) { + continue; + } + ChunkMetadata valueChunkMetadata = + (ChunkMetadata) + alignedChunkMetadata.getValueChunkMetadataList().get(idxInAlignedChunkMetadata); + if (valueChunkMetadata == null) { + continue; + } + Chunk chunk = reader.readMemChunk(valueChunkMetadata); + if (chunk != null) { + chunk + .getHeader() + .setReplaceDecoder( + decoder -> ObjectTypeUtils.getReplaceDecoder(decoder, currentRegionId)); + } + valueChunks.add(chunk); + valuePages.add( + chunk == null + ? null + : new CompactionChunkReader(chunk).readPageDataWithoutUncompressing()); + } + + CompactionAlignedChunkReader alignedChunkReader = + new CompactionAlignedChunkReader(timeChunk, valueChunks, true); + for (int i = 0; i < timePages.size(); i++) { + Pair timePage = timePages.get(i); + List valuePageHeaders = new ArrayList<>(valuePages.size()); + List compressedValuePages = new ArrayList<>(valuePages.size()); + for (int j = 0; j < valuePages.size(); j++) { + Pair valuePage = valuePages.get(j).get(i); + valuePageHeaders.add(valuePage.getLeft()); + compressedValuePages.add(valuePage.getRight()); + } + IPointReader pagePointReader = + alignedChunkReader.getPagePointReader( + timePage.getLeft(), valuePageHeaders, timePage.getRight(), compressedValuePages); + + while (pagePointReader.hasNextTimeValuePair()) { + TimeValuePair timeValuePair = pagePointReader.nextTimeValuePair(); + removeDeletedObjectFiles(timeValuePair, deletionCursors, timeDeletions, objectDeletions); + } + } + } + + private static void removeDeletedObjectFiles( + TimeValuePair timeValuePair, + int[] cursors, + List timeDeletions, + List> objectDeletions) { + long timestamp = timeValuePair.getTimestamp(); + boolean timeDeleted = isDeleted(timestamp, timeDeletions, cursors, 0); + for (int i = 0; i < timeValuePair.getValues().length; i++) { + Binary value = (Binary) timeValuePair.getValues()[i]; + if (value == null) { + continue; + } + if (timeDeleted || isDeleted(timestamp, objectDeletions.get(i), cursors, i + 1)) { + ObjectTypeUtils.deleteObjectPathFromBinary(value); + } + } + } + + private static boolean isDeleted( + long timestamp, List deleteIntervalList, int[] deleteCursors, int idx) { + while (deleteIntervalList != null && deleteCursors[idx] < deleteIntervalList.size()) { + if (deleteIntervalList.get(deleteCursors[idx]).getTimeRange().contains(timestamp)) { + return true; + } else if (deleteIntervalList.get(deleteCursors[idx]).getTimeRange().getMax() < timestamp) { + deleteCursors[idx]++; + } else { + return false; + } + } + return false; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index 1889182a2db4..c07f5a1ad404 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@ -175,7 +175,7 @@ public MultiTsFileDeviceIterator( } } - public boolean hasNextDevice() { + public boolean hasNextDevice() throws IOException { boolean hasNext = false; for (TsFileDeviceIterator iterator : deviceIteratorMap.values()) { hasNext = @@ -196,7 +196,7 @@ public boolean hasNextDevice() { * @return Pair of device full path and whether this device is aligned */ @SuppressWarnings({"squid:S135", "java:S2259"}) - public Pair nextDevice() throws IllegalPathException { + public Pair nextDevice() throws IllegalPathException, IOException { List toBeRemovedResources = new LinkedList<>(); Pair minDevice = null; // get the device from source files sorted from the newest to the oldest by version @@ -275,7 +275,8 @@ public Map getAllSchemasOfCurrentDevice() throws IOEx timeseriesMetadataList, deviceIteratorMap.get(resource).getFirstMeasurementNodeOfCurrentDevice(), schemaMap.keySet(), - true); + true, + null); for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) { if (!schemaMap.containsKey(timeseriesMetadata.getMeasurementId()) && !timeseriesMetadata.getChunkMetadataList().isEmpty()) { @@ -443,7 +444,7 @@ public Map getAllSchemasOfCurrentDevice() throws IOEx */ private void applyModificationForAlignedChunkMetadataList( TsFileResource tsFileResource, List alignedChunkMetadataList) - throws IllegalPathException { + throws IllegalPathException, IOException { if (alignedChunkMetadataList.isEmpty()) { // all the value chunks is empty chunk return; @@ -482,6 +483,16 @@ private void applyModificationForAlignedChunkMetadataList( modificationList.isEmpty() ? Collections.emptyList() : modificationList); } + if (ttlDeletion != null) { + List emptyList = Collections.emptyList(); + CompactionUtils.removeDeletedObjectFiles( + readerMap.get(tsFileResource), + alignedChunkMetadataList, + Collections.singletonList(ttlDeletion), + modificationForValueColumns.stream().map(v -> emptyList).collect(Collectors.toList()), + tsFileResource.getTsFileID().regionId); + } + ModificationUtils.modifyAlignedChunkMetaData( alignedChunkMetadataList, modificationForTimeColumn, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java index 4e71c03d25f4..7a8e884a4913 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java @@ -22,8 +22,11 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.PatternTreeMap; +import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.db.exception.WriteProcessException; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.ModifiedStatus; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.batch.utils.AlignedSeriesBatchCompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.AlignedPageElement; @@ -59,9 +62,11 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class FastAlignedSeriesCompactionExecutor extends SeriesCompactionExecutor { @@ -175,7 +180,8 @@ void deserializeFileIntoChunkMetadataQueue(List fileElements) new ChunkMetadataElement( alignedChunkMetadataList.get(i), i == alignedChunkMetadataList.size() - 1, - fileElement)); + fileElement, + measurementSchemas)); } } } @@ -264,6 +270,24 @@ protected List getAlignedChunkMetadataList(TsFileR } }); + long ttlForTable = + deviceId.isTableModel() + ? DataNodeTTLCache.getInstance() + .getTTLForTable(resource.getDatabaseName(), deviceId.getTableName()) + : Long.MAX_VALUE; + if (ttlForTable != Long.MAX_VALUE) { + ModEntry ttlDeletion = + CompactionUtils.convertTtlToDeletion( + deviceId, CommonDateTimeUtils.currentTime() - ttlForTable); + List emptyList = Collections.emptyList(); + CompactionUtils.removeDeletedObjectFiles( + readerCacheMap.get(resource), + alignedChunkMetadataList, + Collections.singletonList(ttlDeletion), + valueModifications.stream().map(v -> emptyList).collect(Collectors.toList()), + resource.getTsFileID().regionId); + } + // modify aligned chunk metadatas ModificationUtils.modifyAlignedChunkMetaData( alignedChunkMetadataList, timeModifications, valueModifications, ignoreAllNullRows); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java index b07407c7db6f..0a77bf16ea7a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java @@ -52,6 +52,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -160,7 +161,10 @@ void deserializeFileIntoChunkMetadataQueue(List fileElements) // add into queue chunkMetadataQueue.add( new ChunkMetadataElement( - chunkMetadata, i == iChunkMetadataList.size() - 1, fileElement)); + chunkMetadata, + i == iChunkMetadataList.size() - 1, + fileElement, + Collections.emptyList())); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java index 6828841f1bda..2d29e3944d85 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/element/ChunkMetadataElement.java @@ -23,6 +23,7 @@ import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.common.Chunk; +import org.apache.tsfile.write.schema.IMeasurementSchema; import java.util.List; @@ -42,12 +43,19 @@ public class ChunkMetadataElement { public boolean needForceDecodingPage; + // for aligned series + public List measurementSchemasOfValueChunks; + public ChunkMetadataElement( - IChunkMetadata chunkMetadata, boolean isLastChunk, FileElement fileElement) { + IChunkMetadata chunkMetadata, + boolean isLastChunk, + FileElement fileElement, + List measurementSchemasOfValueChunks) { this.chunkMetadata = chunkMetadata; this.startTime = chunkMetadata.getStartTime(); this.isLastChunk = isLastChunk; this.fileElement = fileElement; + this.measurementSchemasOfValueChunks = measurementSchemasOfValueChunks; } public void clearChunks() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionAlignedChunkReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionAlignedChunkReader.java index a94150deca19..3cbc0c2fbae1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionAlignedChunkReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/reader/CompactionAlignedChunkReader.java @@ -92,7 +92,7 @@ public IPointReader getPagePointReader( ByteBuffer compressedTimePageData, List compressedValuePageDatas) throws IOException { - return getPontReader( + return getPointReader( timePageHeader, valuePageHeaders, compressedTimePageData, @@ -106,11 +106,11 @@ public IPointReader getBatchedPagePointReader( ByteBuffer compressedTimePageData, List compressedValuePageDatas) throws IOException { - return getPontReader( + return getPointReader( timePageHeader, valuePageHeaders, compressedTimePageData, compressedValuePageDatas, false); } - private IPointReader getPontReader( + private IPointReader getPointReader( PageHeader timePageHeader, List valuePageHeaders, ByteBuffer compressedTimePageData, @@ -146,7 +146,7 @@ private IPointReader getPontReader( valuePageHeaders.get(i), uncompressedPageData, valueType, - Decoder.getDecoderByType(valueChunkHeader.getEncodingType(), valueType)); + valueChunkHeader.calculateDecoderForNonTimeChunk()); valuePageReader.setDeleteIntervalList(valueDeleteIntervalList.get(i)); valuePageReaders.add(valuePageReader); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java index 928c0cdbc6b2..c2e0959e6523 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java @@ -565,6 +565,7 @@ private long estimateMemorySizeAsPageWriter(PageLoader pageLoader) { case TEXT: case STRING: case BLOB: + case OBJECT: size = pageLoader.getHeader().getUncompressedSize(); break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java index b09adb17bba8..cff44c7618a0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java @@ -322,6 +322,7 @@ private void writeTimeAndValueToChunkWriter(TimeValuePair timeValuePair) { case TEXT: case BLOB: case STRING: + case OBJECT: chunkWriter.write(timeValuePair.getTimestamp(), timeValuePair.getValue().getBinary()); break; case FLOAT: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java index 3458f9870d46..f3cd5185b58a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/writer/AbstractCompactionWriter.java @@ -149,6 +149,7 @@ protected void writeDataPoint(long timestamp, TsPrimitiveType value, IChunkWrite case TEXT: case STRING: case BLOB: + case OBJECT: chunkWriterImpl.write(timestamp, value.getBinary()); break; case DOUBLE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java index d515a7fa3a90..d0f3426423ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java @@ -283,7 +283,7 @@ private void checkNonAlignedDeviceSeries( throws IOException { List timeseriesMetadataList = new ArrayList<>(); reader.getDeviceTimeseriesMetadata( - timeseriesMetadataList, metadataIndexNode, Collections.emptySet(), true); + timeseriesMetadataList, metadataIndexNode, Collections.emptySet(), true, null); long actualDeviceStartTime = Long.MAX_VALUE; long actualDeviceEndTime = Long.MIN_VALUE; for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java index e69a5b8b8c39..25c4a876ab7d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedReadOnlyMemChunk.java @@ -220,6 +220,7 @@ public void initChunkMetaFromTvLists(Filter globalTimeFilter) { case TEXT: case BLOB: case STRING: + case OBJECT: for (int i = 0; i < tsBlock.getPositionCount(); i++) { if (tsBlock.getColumn(column).isNull(i)) { continue; @@ -412,6 +413,7 @@ private void writeValidValuesIntoTsBlock(TsBlockBuilder builder) throws IOExcept case TEXT: case BLOB: case STRING: + case OBJECT: valueBuilder.writeBinary(values[columnIndex].getBinary()); break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java index 49f8ab0dadab..f42867d4772c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/AlignedWritableMemChunk.java @@ -696,6 +696,7 @@ private void handleEncoding( case TEXT: case STRING: case BLOB: + case OBJECT: alignedChunkWriter.writeByColumn( time, isNull ? null : list.getBinaryByValueIndex(originRowIndex, columnIndex), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java index dbc3183df80c..6499d33fcd9f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java @@ -166,6 +166,7 @@ public void writeNonAlignedTablet( case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] binaryValues = (Binary[]) valueList; putBinaries(times, binaryValues, bitMap, start, end); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java index a7c6eb96d429..27883b34e9ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskAlignedChunkLoader.java @@ -24,7 +24,9 @@ import org.apache.iotdb.db.storageengine.buffer.ChunkCache; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.utils.ObjectTypeUtils; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -92,7 +94,7 @@ public IChunkReader getChunkReader(IChunkMetadata chunkMetaData, Filter globalTi context); List valueChunkList = new ArrayList<>(); for (IChunkMetadata valueChunkMetadata : alignedChunkMetadata.getValueChunkMetadataList()) { - valueChunkList.add( + Chunk chunk = valueChunkMetadata == null ? null : ChunkCache.getInstance() @@ -104,7 +106,17 @@ public IChunkReader getChunkReader(IChunkMetadata chunkMetaData, Filter globalTi resource.isClosed()), valueChunkMetadata.getDeleteIntervalList(), valueChunkMetadata.getStatistics(), - context)); + context); + final TsFileID tsFileID = getTsFileID(); + if (chunk != null + && tsFileID.regionId > 0 + && chunkMetaData.getDataType() == TSDataType.OBJECT) { + chunk + .getHeader() + .setReplaceDecoder( + decoder -> ObjectTypeUtils.getReplaceDecoder(decoder, tsFileID.regionId)); + } + valueChunkList.add(chunk); } long t2 = System.nanoTime(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java index be33428ae65f..a2ff24233d9f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java @@ -24,7 +24,9 @@ import org.apache.iotdb.db.storageengine.buffer.ChunkCache; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.utils.ObjectTypeUtils; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.common.Chunk; @@ -85,6 +87,14 @@ public IChunkReader getChunkReader(IChunkMetadata chunkMetaData, Filter globalTi chunkMetaData.getStatistics(), context); + final TsFileID tsFileID = getTsFileID(); + if (tsFileID.regionId > 0 && chunkMetaData.getDataType() == TSDataType.OBJECT) { + chunk + .getHeader() + .setReplaceDecoder( + decoder -> ObjectTypeUtils.getReplaceDecoder(decoder, tsFileID.regionId)); + } + long t2 = System.nanoTime(); IChunkReader chunkReader = new ChunkReader(chunk, globalTimeFilter); SeriesScanCostMetricSet.getInstance() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java index 7ee43a93aea8..dbe948c01b86 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemAlignedPageReader.java @@ -293,6 +293,7 @@ private void updatePageStatisticsFromTsBlock( break; case TEXT: case BLOB: + case OBJECT: case STRING: for (int i = 0; i < tsBlock.getPositionCount(); i++) { valueStatistics[column].update( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java index a86802400722..e7427ef87d8d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/MemPageReader.java @@ -105,6 +105,7 @@ public BatchData getAllSatisfiedPageData(boolean ascending) throws IOException { case TEXT: case STRING: case BLOB: + case OBJECT: batchData.putBinary( tsBlock.getTimeColumn().getLong(i), tsBlock.getColumn(0).getBinary(i)); break; @@ -269,6 +270,7 @@ private void updatePageStatisticsFromTsBlock(Statistics statistics) { case TEXT: case BLOB: case STRING: + case OBJECT: for (int i = 0; i < tsBlock.getPositionCount(); i++) { statistics.update(tsBlock.getTimeByIndex(i), tsBlock.getColumn(0).getBinary(i)); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index 8cc080016ca4..a7d79f92b575 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.service.metrics.WritingMetrics; import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.Checkpoint; @@ -332,6 +333,8 @@ private void handleInfoEntry(WALEntry walEntry) { searchIndex = ((DeleteDataNode) walEntry.getValue()).getSearchIndex(); } else if (walEntry.getType() == WALEntryType.RELATIONAL_DELETE_DATA_NODE) { searchIndex = ((RelationalDeleteDataNode) walEntry.getValue()).getSearchIndex(); + } else if (walEntry.getType() == WALEntryType.OBJECT_FILE_NODE) { + searchIndex = ((ObjectNode) walEntry.getValue()).getSearchIndex(); } else { searchIndex = ((InsertNode) walEntry.getValue()).getSearchIndex(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java index f7b85089febe..4c0ff8809d26 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntry.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; @@ -77,6 +78,8 @@ protected WALEntry(long memTableId, WALEntryValue value, boolean wait) { this.type = WALEntryType.CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE; } else if (value instanceof RelationalDeleteDataNode) { this.type = WALEntryType.RELATIONAL_DELETE_DATA_NODE; + } else if (value instanceof ObjectNode) { + this.type = WALEntryType.OBJECT_FILE_NODE; } else { throw new RuntimeException("Unknown WALEntry type"); } @@ -134,6 +137,9 @@ public static WALEntry deserialize(DataInputStream stream) throws IOException { case CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE: value = (ContinuousSameSearchIndexSeparatorNode) PlanNodeType.deserializeFromWAL(stream); break; + case OBJECT_FILE_NODE: + value = (ObjectNode) PlanNodeType.deserializeFromWAL(stream); + break; default: throw new RuntimeException("Unknown WALEntry type " + type); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntryType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntryType.java index 1c3ddcbf702a..829affd1b4e1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntryType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALEntryType.java @@ -45,6 +45,7 @@ public enum WALEntryType { /** {@link org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable} */ MEMORY_TABLE_SNAPSHOT((byte) 10), RELATIONAL_DELETE_DATA_NODE((byte) 11), + OBJECT_FILE_NODE((byte) 12), // endregion // region signal entry type // signal wal buffer has been closed @@ -71,7 +72,8 @@ public boolean needSearch() { || this == INSERT_ROW_NODE || this == INSERT_ROWS_NODE || this == DELETE_DATA_NODE - || this == RELATIONAL_DELETE_DATA_NODE; + || this == RELATIONAL_DELETE_DATA_NODE + || this == OBJECT_FILE_NODE; } public static WALEntryType valueOf(byte code) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java index 6da50edba4f8..791ef63a7525 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALInfoEntry.java @@ -23,6 +23,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALMode; @@ -89,6 +90,7 @@ public void serialize(IWALByteBufferView buffer) { case RELATIONAL_DELETE_DATA_NODE: case MEMORY_TABLE_SNAPSHOT: case CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE: + case OBJECT_FILE_NODE: value.serializeToWAL(buffer); break; case MEMORY_TABLE_CHECKPOINT: @@ -166,6 +168,8 @@ public long getMemorySize() { case CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE: case MEMORY_TABLE_CHECKPOINT: return RamUsageEstimator.sizeOfObject(value); + case OBJECT_FILE_NODE: + return ((ObjectNode) value).serializedSize(); default: throw new RuntimeException("Unsupported wal entry type " + type); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java index 0d0e0a527f0f..a8fbbee0dc4f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/IWALNode.java @@ -26,6 +26,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.storageengine.dataregion.flush.FlushListener; import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; @@ -54,6 +55,8 @@ public interface IWALNode extends FlushListener, AutoCloseable, ConsensusReqRead /** Log BatchDoneNode */ WALFlushListener log(long memTableId, ContinuousSameSearchIndexSeparatorNode separatorNode); + WALFlushListener log(long memTableId, ObjectNode objectNode); + /** Callback when memTable created. */ void onMemTableCreated(IMemTable memTable, String targetTsFile); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java index 48bf4c1cfb99..e35d5e79fc01 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALFakeNode.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALException; @@ -82,6 +83,11 @@ public WALFlushListener log( return getResult(); } + @Override + public WALFlushListener log(long memTableId, ObjectNode objectNode) { + return getResult(); + } + private WALFlushListener getResult() { switch (status) { case SUCCESS: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java index 8803aca56c44..07dd4d78f660 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.ObjectNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.service.metrics.WritingMetrics; import org.apache.iotdb.db.storageengine.StorageEngine; @@ -62,6 +63,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -190,6 +193,12 @@ public WALFlushListener log( return log(walEntry); } + @Override + public WALFlushListener log(long memTableId, ObjectNode objectNode) { + WALEntry walEntry = new WALInfoEntry(memTableId, objectNode); + return log(walEntry); + } + private WALFlushListener log(WALEntry walEntry) { buffer.write(walEntry); @@ -646,6 +655,8 @@ public boolean hasNext() { AtomicBoolean notFirstFile = new AtomicBoolean(false); AtomicBoolean hasCollectedSufficientData = new AtomicBoolean(false); + long memorySize = 0; + // try to collect current tmpNodes to insertNodes, return true if successfully collect an // insert node Runnable tryToCollectInsertNodeAndBumpIndex = @@ -684,7 +695,23 @@ public boolean hasNext() { } else if (currentWalEntryIndex < nextSearchIndex) { // WAL entry is outdated, do nothing, continue to see next WAL entry } else if (currentWalEntryIndex == nextSearchIndex) { - tmpNodes.get().add(new IoTConsensusRequest(buffer)); + if (type == WALEntryType.OBJECT_FILE_NODE) { + WALEntry walEntry = + WALEntry.deserialize( + new DataInputStream(new ByteArrayInputStream(buffer.array()))); + // only be called by leader read from wal + // wal only has relativePath, offset, eof, length + // need to add WALEntryType + memtableId + relativePath, offset, eof, length + + // content + // need to add IoTConsensusRequest instead of ObjectNode + tmpNodes + .get() + .add(new IoTConsensusRequest(((ObjectNode) walEntry.getValue()).serialize())); + memorySize += ((ObjectNode) walEntry.getValue()).getMemorySize(); + } else { + tmpNodes.get().add(new IoTConsensusRequest(buffer)); + memorySize += buffer.remaining(); + } } else { // currentWalEntryIndex > targetIndex // WAL entry of targetIndex has been fully collected, put them into insertNodes @@ -696,12 +723,31 @@ public boolean hasNext() { currentWalEntryIndex); nextSearchIndex = currentWalEntryIndex; } - tmpNodes.get().add(new IoTConsensusRequest(buffer)); + if (type == WALEntryType.OBJECT_FILE_NODE) { + WALEntry walEntry = + WALEntry.deserialize( + new DataInputStream(new ByteArrayInputStream(buffer.array()))); + // only be called by leader read from wal + // wal only has relativePath, offset, eof, length + // need to add WALEntryType + memtableId + relativePath, offset, eof, length + + // content + // need to add IoTConsensusRequest instead of ObjectNode + tmpNodes + .get() + .add(new IoTConsensusRequest(((ObjectNode) walEntry.getValue()).serialize())); + memorySize += ((ObjectNode) walEntry.getValue()).getMemorySize(); + } else { + tmpNodes.get().add(new IoTConsensusRequest(buffer)); + memorySize += buffer.remaining(); + } } } else { tryToCollectInsertNodeAndBumpIndex.run(); } - if (hasCollectedSufficientData.get()) { + if (memorySize > config.getWalBufferSize()) { + tryToCollectInsertNodeAndBumpIndex.run(); + } + if (memorySize > config.getWalBufferSize() || hasCollectedSufficientData.get()) { break COLLECT_FILE_LOOP; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java index 798f25d4e266..f4da41073345 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java @@ -223,6 +223,8 @@ public void redoLog(WALEntry walEntry) { case CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE: // The CONTINUOUS_SAME_SEARCH_INDEX_SEPARATOR_NODE doesn't need redo break; + case OBJECT_FILE_NODE: + break; default: throw new RuntimeException("Unsupported type " + walEntry.getType()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java index be31a03d76ae..6e8d3850b8e2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/BatchedAlignedValueChunkData.java @@ -109,6 +109,7 @@ public void writeDecodeValuePage(long[] times, TsPrimitiveType[] values, TSDataT break; case TEXT: case BLOB: + case OBJECT: case STRING: dataSize += ReadWriteIOUtils.write(values[i].getBinary(), stream); break; @@ -199,6 +200,7 @@ private void buildValueChunkWriter( break; case TEXT: case BLOB: + case OBJECT: case STRING: final Binary binaryValue = isNull ? DEFAULT_BINARY : ReadWriteIOUtils.readBinary(stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java index d929f7656077..a5fa8b54e7b1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java @@ -45,6 +45,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -65,12 +66,16 @@ public class TierManager { */ private final List unSeqTiers = new ArrayList<>(); + private final List objectTiers = new ArrayList<>(); + /** seq file folder's rawFsPath path -> tier level */ private final Map seqDir2TierLevel = new HashMap<>(); /** unSeq file folder's rawFsPath path -> tier level */ private final Map unSeqDir2TierLevel = new HashMap<>(); + private List objectDirs; + /** total space of each tier, Long.MAX_VALUE when one tier contains remote storage */ private long[] tierDiskTotalSpace; @@ -151,6 +156,22 @@ public synchronized void initFolders() { for (String dir : unSeqDirs) { unSeqDir2TierLevel.put(dir, tierLevel); } + + objectDirs = + Arrays.stream(tierDirs[tierLevel]) + .filter(Objects::nonNull) + .map( + v -> + FSFactoryProducer.getFSFactory() + .getFile(v, IoTDBConstant.OBJECT_FOLDER_NAME) + .getPath()) + .collect(Collectors.toList()); + + try { + objectTiers.add(new FolderManager(objectDirs, directoryStrategyType)); + } catch (DiskSpaceInsufficientException e) { + logger.error("All disks of tier {} are full.", tierLevel, e); + } } tierDiskTotalSpace = getTierDiskSpace(DiskSpaceType.TOTAL); @@ -160,6 +181,7 @@ public synchronized void resetFolders() { long startTime = System.currentTimeMillis(); seqTiers.clear(); unSeqTiers.clear(); + objectTiers.clear(); seqDir2TierLevel.clear(); unSeqDir2TierLevel.clear(); @@ -190,6 +212,10 @@ public String getNextFolderForTsFile(int tierLevel, boolean sequence) : unSeqTiers.get(tierLevel).getNextFolder(); } + public String getNextFolderForObjectFile() throws DiskSpaceInsufficientException { + return objectTiers.get(0).getNextFolder(); + } + public FolderManager getFolderManager(int tierLevel, boolean sequence) { return sequence ? seqTiers.get(tierLevel) : unSeqTiers.get(tierLevel); } @@ -222,6 +248,30 @@ public List getAllLocalUnSequenceFileFolders() { .collect(Collectors.toList()); } + public List getAllObjectFileFolders() { + return objectDirs; + } + + public Optional getAbsoluteObjectFilePath(String filePath) { + return getAbsoluteObjectFilePath(filePath, false); + } + + public Optional getAbsoluteObjectFilePath(String filePath, boolean needTempFile) { + for (String objectDir : objectDirs) { + File objectFile = FSFactoryProducer.getFSFactory().getFile(objectDir, filePath); + if (objectFile.exists()) { + return Optional.of(objectFile); + } + if (needTempFile) { + if (new File(objectFile.getPath() + ".tmp").exists() + || new File(objectFile.getPath() + ".back").exists()) { + return Optional.of(objectFile); + } + } + } + return Optional.empty(); + } + public int getTiersNum() { return seqTiers.size(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java index 3281b67c034d..dbf3b989800e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java @@ -343,6 +343,7 @@ public static Object createDataListsByType(TSDataType dataType, int size) { case TEXT: case STRING: case BLOB: + case OBJECT: Binary[][] binaries = new Binary[arrayNumber][]; for (int i = 0; i < arrayNumber; i++) { binaries[i] = new Binary[ARRAY_SIZE]; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java index f50e472c414f..38686bee0477 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/TsFileSplitByPartitionTool.java @@ -430,6 +430,7 @@ protected void rewritePageIntoFiles( case TEXT: case BLOB: case STRING: + case OBJECT: chunkWriter.write(time, (Binary) value); break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java index b9dbcfe86853..1fa1ec44fc18 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/CommonUtils.java @@ -380,6 +380,7 @@ public static Object createValueColumnOfDataType( case TEXT: case STRING: case BLOB: + case OBJECT: valueColumn = new Binary[rowNum]; break; case DATE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java index 402921338a49..d9d4cecc4e42 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/EncodingInferenceUtils.java @@ -51,6 +51,7 @@ public static TSEncoding getDefaultEncoding(TSDataType dataType) { case TEXT: case BLOB: case STRING: + case OBJECT: return conf.getDefaultTextEncoding(); default: throw new UnSupportedDataTypeException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java index 843bcdb8937c..1de1422b07be 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java @@ -108,7 +108,9 @@ public static long getBinaryColumnSize(Binary[] column, int start, int end, TSSt if (results == null || results[i] == null || results[i].code == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - memSize += RamUsageEstimator.sizeOf(column[i].getValues()); + if (column[i] != null) { + memSize += RamUsageEstimator.sizeOf(column[i].getValues()); + } } } return memSize; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java new file mode 100644 index 000000000000..0d3d9f963bbc --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectTypeUtils.java @@ -0,0 +1,308 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; +import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; +import org.apache.iotdb.commons.exception.IoTDBRuntimeException; +import org.apache.iotdb.commons.exception.ObjectFileNotExist; +import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.queryengine.plan.Coordinator; +import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; +import org.apache.iotdb.db.service.metrics.FileMetrics; +import org.apache.iotdb.db.storageengine.dataregion.Base32ObjectPath; +import org.apache.iotdb.db.storageengine.dataregion.IObjectPath; +import org.apache.iotdb.db.storageengine.dataregion.PlainObjectPath; +import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; +import org.apache.iotdb.mpp.rpc.thrift.TReadObjectReq; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.encoding.decoder.Decoder; +import org.apache.tsfile.encoding.decoder.DecoderWrapper; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +public class ObjectTypeUtils { + + private static final Logger logger = LoggerFactory.getLogger(ObjectTypeUtils.class); + private static final TierManager TIER_MANAGER = TierManager.getInstance(); + + private ObjectTypeUtils() {} + + public static ByteBuffer readObjectContent( + Binary binary, long offset, int length, boolean mayNotInCurrentNode) { + Pair objectLengthPathPair = + ObjectTypeUtils.parseObjectBinaryToSizeStringPathPair(binary); + long fileLength = objectLengthPathPair.getLeft(); + String relativePath = objectLengthPathPair.getRight(); + int actualReadSize = + ObjectTypeUtils.getActualReadSize( + relativePath, fileLength, offset, length < 0 ? fileLength : length); + return ObjectTypeUtils.readObjectContent( + relativePath, offset, actualReadSize, mayNotInCurrentNode); + } + + public static ByteBuffer readObjectContent( + String relativePath, long offset, int readSize, boolean mayNotInCurrentNode) { + Optional objectFile = TIER_MANAGER.getAbsoluteObjectFilePath(relativePath, false); + if (objectFile.isPresent()) { + return readObjectContentFromLocalFile(objectFile.get(), offset, readSize); + } + if (mayNotInCurrentNode) { + return readObjectContentFromRemoteFile(relativePath, offset, readSize); + } + throw new ObjectFileNotExist(relativePath); + } + + private static ByteBuffer readObjectContentFromLocalFile(File file, long offset, long readSize) { + byte[] bytes = new byte[(int) readSize]; + ByteBuffer buffer = ByteBuffer.wrap(bytes); + try (FileChannel fileChannel = FileChannel.open(file.toPath(), StandardOpenOption.READ)) { + fileChannel.read(buffer, offset); + } catch (IOException e) { + throw new IoTDBRuntimeException(e, TSStatusCode.OBJECT_READ_ERROR.getStatusCode()); + } + buffer.flip(); + return buffer; + } + + private static ByteBuffer readObjectContentFromRemoteFile( + final String relativePath, final long offset, final int readSize) { + int regionId; + try { + regionId = Integer.parseInt(Paths.get(relativePath).getName(0).toString()); + } catch (NumberFormatException e) { + throw new IoTDBRuntimeException( + "wrong object file path: " + relativePath, + TSStatusCode.OBJECT_READ_ERROR.getStatusCode()); + } + TConsensusGroupId consensusGroupId = + new TConsensusGroupId(TConsensusGroupType.DataRegion, regionId); + List regionReplicaSetList = + ClusterPartitionFetcher.getInstance() + .getRegionReplicaSet(Collections.singletonList(consensusGroupId)); + if (regionReplicaSetList.isEmpty()) { + throw new ObjectFileNotExist(relativePath); + } + TRegionReplicaSet regionReplicaSet = regionReplicaSetList.iterator().next(); + if (regionReplicaSet.getDataNodeLocations().isEmpty()) { + throw new ObjectFileNotExist(relativePath); + } + final int batchSize = 1024 * 1024; + final TReadObjectReq req = new TReadObjectReq(); + req.setRelativePath(relativePath); + ByteBuffer buffer = ByteBuffer.allocate(readSize); + for (int i = 0; i < regionReplicaSet.getDataNodeLocations().size(); i++) { + TDataNodeLocation dataNodeLocation = regionReplicaSet.getDataNodeLocations().get(i); + int toReadSizeInCurrentDataNode = readSize; + try (SyncDataNodeInternalServiceClient client = + Coordinator.getInstance() + .getInternalServiceClientManager() + .borrowClient(dataNodeLocation.getInternalEndPoint())) { + while (toReadSizeInCurrentDataNode > 0) { + req.setOffset(offset + buffer.position()); + req.setSize(Math.min(toReadSizeInCurrentDataNode, batchSize)); + toReadSizeInCurrentDataNode -= req.getSize(); + ByteBuffer partial = client.readObject(req); + buffer.put(partial); + } + } catch (Exception e) { + logger.warn("Failed to read object from datanode: {}", dataNodeLocation, e); + if (i == regionReplicaSet.getDataNodeLocations().size() - 1) { + throw new IoTDBRuntimeException(e, TSStatusCode.OBJECT_READ_ERROR.getStatusCode()); + } + continue; + } + break; + } + buffer.flip(); + return buffer; + } + + public static Binary generateObjectBinary(long objectSize, IObjectPath objectPath) { + byte[] valueBytes = new byte[objectPath.getSerializeSizeToObjectValue() + Long.BYTES]; + ByteBuffer buffer = ByteBuffer.wrap(valueBytes); + ReadWriteIOUtils.write(objectSize, buffer); + objectPath.serializeToObjectValue(buffer); + return new Binary(buffer.array()); + } + + public static DecoderWrapper getReplaceDecoder(final Decoder decoder, final int newRegionId) { + return new ObjectRegionIdReplaceDecoder(decoder, newRegionId); + } + + private static class ObjectRegionIdReplaceDecoder extends DecoderWrapper { + + private final int newRegionId; + + public ObjectRegionIdReplaceDecoder(Decoder decoder, int newRegionId) { + super(decoder); + this.newRegionId = newRegionId; + } + + @Override + public Binary readBinary(ByteBuffer buffer) { + Binary originValue = originDecoder.readBinary(buffer); + return ObjectTypeUtils.replaceRegionIdForObjectBinary(newRegionId, originValue); + } + } + + public static Binary replaceRegionIdForObjectBinary(int newRegionId, Binary originValue) { + Pair pair = + ObjectTypeUtils.parseObjectBinaryToSizeIObjectPathPair(originValue); + IObjectPath objectPath = pair.getRight(); + try { + Path path; + if (objectPath instanceof PlainObjectPath) { + path = Paths.get(objectPath.toString()); + } else { + path = ((Base32ObjectPath) objectPath).getPath(); + } + int regionId = Integer.parseInt(path.getName(0).toString()); + if (regionId == newRegionId) { + return originValue; + } + IObjectPath newObjectPath; + if (objectPath instanceof PlainObjectPath) { + String newPath = objectPath.toString().replaceFirst(regionId + "", newRegionId + ""); + newObjectPath = new PlainObjectPath(newPath); + } else { + String[] subPath = new String[path.getNameCount() - 1]; + for (int i = 1; i < path.getNameCount(); i++) { + subPath[i - 1] = path.getName(i).toString(); + } + Path newPath = Paths.get(newRegionId + "", subPath); + newObjectPath = new Base32ObjectPath(newPath); + } + return ObjectTypeUtils.generateObjectBinary(pair.getLeft(), newObjectPath); + } catch (NumberFormatException e) { + throw new IoTDBRuntimeException( + "wrong object file path: " + pair.getRight(), + TSStatusCode.OBJECT_READ_ERROR.getStatusCode()); + } + } + + public static int getActualReadSize(String filePath, long fileSize, long offset, long length) { + if (offset < 0) { + throw new SemanticException(String.format("offset %d is less than 0.", offset)); + } + if (offset >= fileSize) { + throw new SemanticException( + String.format( + "offset %d is greater than or equal to object size %d, file path is %s", + offset, fileSize, filePath)); + } + long actualReadSize = Math.min(length < 0 ? fileSize : length, fileSize - offset); + if (actualReadSize > Integer.MAX_VALUE) { + throw new SemanticException( + String.format( + "Read object size %s is too large (size > 2G), file path is %s", + actualReadSize, filePath)); + } + return (int) actualReadSize; + } + + public static Pair parseObjectBinaryToSizeStringPathPair(Binary binary) { + byte[] bytes = binary.getValues(); + ByteBuffer buffer = ByteBuffer.wrap(bytes); + long length = buffer.getLong(); + String relativeObjectFilePath = + IObjectPath.getDeserializer().deserializeFromObjectValue(buffer).toString(); + return new Pair<>(length, relativeObjectFilePath); + } + + public static Pair parseObjectBinaryToSizeIObjectPathPair(Binary binary) { + byte[] bytes = binary.getValues(); + ByteBuffer buffer = ByteBuffer.wrap(bytes); + long length = buffer.getLong(); + IObjectPath objectPath = IObjectPath.getDeserializer().deserializeFromObjectValue(buffer); + return new Pair<>(length, objectPath); + } + + public static long getObjectLength(Binary binary) { + byte[] bytes = binary.getValues(); + ByteBuffer wrap = ByteBuffer.wrap(bytes); + return wrap.getLong(); + } + + public static Optional getObjectPathFromBinary(Binary binary) { + byte[] bytes = binary.getValues(); + String relativeObjectFilePath = + new String(bytes, 8, bytes.length - 8, TSFileConfig.STRING_CHARSET); + return TIER_MANAGER.getAbsoluteObjectFilePath(relativeObjectFilePath); + } + + public static Optional getNullableObjectPathFromBinary( + Binary binary, boolean needTempFile) { + byte[] bytes = binary.getValues(); + ByteBuffer buffer = ByteBuffer.wrap(bytes, 8, bytes.length - 8); + String relativeObjectFilePath = + IObjectPath.getDeserializer().deserializeFromObjectValue(buffer).toString(); + return TIER_MANAGER.getAbsoluteObjectFilePath(relativeObjectFilePath, needTempFile); + } + + public static void deleteObjectPathFromBinary(Binary binary) { + Optional file = ObjectTypeUtils.getNullableObjectPathFromBinary(binary, true); + if (!file.isPresent()) { + return; + } + File tmpFile = new File(file.get().getPath() + ".tmp"); + File bakFile = new File(file.get().getPath() + ".back"); + for (int i = 0; i < 2; i++) { + if (file.get().exists()) { + FileMetrics.getInstance().decreaseObjectFileNum(1); + FileMetrics.getInstance().decreaseObjectFileSize(file.get().length()); + } + try { + deleteObjectFile(file.get()); + deleteObjectFile(tmpFile); + deleteObjectFile(bakFile); + } catch (IOException e) { + logger.error("Failed to remove object file {}", file.get().getAbsolutePath(), e); + } + } + } + + private static void deleteObjectFile(File file) throws IOException { + if (file.exists()) { + logger.info("Remove object file {}, size is {}(byte)", file.getAbsolutePath(), file.length()); + } + Files.deleteIfExists(file.toPath()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java new file mode 100644 index 000000000000..e007027bd6d8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.utils; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; + +import org.apache.tsfile.external.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; + +public class ObjectWriter implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(ObjectWriter.class); + + private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private final FileOutputStream fos; + + private final File file; + + public ObjectWriter(File filePath) throws FileNotFoundException { + try { + FileUtils.forceMkdir(filePath.getParentFile()); + } catch (final IOException e) { + throw new FileNotFoundException("Error occurred during creating directory " + filePath); + } + if (!Files.exists(filePath.toPath())) { + try { + Files.createFile(filePath.toPath()); + } catch (IOException e) { + throw new FileNotFoundException(e.getMessage()); + } + } + file = filePath; + fos = new FileOutputStream(filePath, true); + } + + public void write(boolean isGeneratedByConsensus, long offset, byte[] content) + throws IOException { + if (file.length() != offset) { + if (isGeneratedByConsensus || offset == 0) { + fos.getChannel().truncate(offset); + } else { + throw new IOException( + "The file length " + file.length() + " is not equal to the offset " + offset); + } + } + if (file.length() + content.length > config.getMaxObjectSizeInByte()) { + throw new IOException("The file length is larger than max_object_file_size_in_bytes"); + } + fos.write(content); + } + + @Override + public void close() throws Exception { + fos.close(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java index cd4ca79ac438..6220623dd3b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/QueryDataSetUtils.java @@ -51,6 +51,9 @@ public class QueryDataSetUtils { private static final TSFileConfig TSFLE_CONFIG = TSFileDescriptor.getInstance().getConfig(); + // default return 8 MB each time + private static final long MAX_RETURN_SIZE = 8 * 1024 * 1024; + private QueryDataSetUtils() {} public static Pair convertTsBlockByFetchSize( @@ -236,6 +239,7 @@ public static TSQueryDataSet convertTsBlockByFetchSize(List tsBlocks) case TEXT: case BLOB: case STRING: + case OBJECT: for (int i = 0; i < currentCount; i++) { rowCount++; if (column.isNull(i)) { @@ -379,6 +383,7 @@ private static void serializeTsBlock( case TEXT: case BLOB: case STRING: + case OBJECT: doWithTextColumn( rowCount, column, @@ -614,8 +619,9 @@ public static Pair, Boolean> convertQueryResultByFetchSize( IQueryExecution queryExecution, int fetchSize) throws IoTDBException { fetchSize = fetchSize > 0 ? fetchSize : TSFLE_CONFIG.getMaxTsBlockLineNumber(); int rowCount = 0; + long memorySize = 0; List res = new ArrayList<>(); - while (rowCount < fetchSize) { + while (rowCount < fetchSize && memorySize < MAX_RETURN_SIZE) { Optional optionalByteBuffer = queryExecution.getByteBufferBatchResult(); if (!optionalByteBuffer.isPresent()) { break; @@ -632,6 +638,7 @@ public static Pair, Boolean> convertQueryResultByFetchSize( res.add(byteBuffer); } rowCount += positionCount; + memorySize += byteBuffer.limit(); } return new Pair<>(res, !queryExecution.hasNextResult()); } @@ -753,6 +760,7 @@ public static Object[] readTabletValuesFromBuffer( case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] binaryValues = new Binary[size]; for (int index = 0; index < size; index++) { int binarySize = buffer.getInt(); @@ -795,6 +803,7 @@ public static Object[] readTabletValuesFromStream( case TEXT: case BLOB: case STRING: + case OBJECT: parseTextColumn(size, stream, values, i); break; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TabletDecoder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TabletDecoder.java index 335d7a23b42a..bd1eba9cfa8f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TabletDecoder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TabletDecoder.java @@ -192,6 +192,7 @@ private Object decodeColumn(ByteBuffer uncompressed, int columnIndex) { case STRING: case BLOB: case TEXT: + case OBJECT: Binary[] binaryCol = new Binary[rowSize]; if (encoding == TSEncoding.PLAIN) { // PlainEncoder uses var int, which may cause compatibility problem diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java index e11b586e7e42..a66e7d7b856b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeValuePairUtils.java @@ -62,6 +62,7 @@ public static void setTimeValuePair(TimeValuePair from, TimeValuePair to) { case TEXT: case BLOB: case STRING: + case OBJECT: to.getValue().setBinary(from.getValue().getBinary()); break; case BOOLEAN: @@ -89,6 +90,7 @@ public static TimeValuePair getEmptyTimeValuePair(TSDataType dataType) { case TEXT: case BLOB: case STRING: + case OBJECT: return new TimeValuePair(0, new TsBinary(new Binary("", TSFileConfig.STRING_CHARSET))); default: throw new UnsupportedOperationException("Unrecognized datatype: " + dataType); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java index cca6cd86a6ed..aa2855f7b059 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TypeInferenceUtils.java @@ -325,6 +325,7 @@ public static boolean canAutoCast(TSDataType fromType, TSDataType toType) { case DATE: case TIMESTAMP: case BLOB: + case OBJECT: case STRING: return false; default: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 4be8e9f437e7..844adfb0e512 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -204,6 +204,7 @@ public synchronized void putAlignedValue(long timestamp, Object[] value) { case TEXT: case BLOB: case STRING: + case OBJECT: ((Binary[]) columnValues.get(arrayIndex))[elementIndex] = columnValue != null ? (Binary) columnValue : Binary.EMPTY_VALUE; memoryBinaryChunkSize[i] += @@ -304,6 +305,7 @@ private TsPrimitiveType getAlignedValueByValueIndex( case TEXT: case BLOB: case STRING: + case OBJECT: Binary valueT = ((Binary[]) columnValues.get(arrayIndex))[elementIndex]; vector[columnIndex] = TsPrimitiveType.getByType(TSDataType.TEXT, valueT); break; @@ -365,6 +367,7 @@ public void extendColumn(TSDataType dataType) { case TEXT: case STRING: case BLOB: + case OBJECT: columnValue.add(getPrimitiveArraysByType(TSDataType.TEXT)); break; case FLOAT: @@ -642,6 +645,7 @@ protected Object cloneValue(TSDataType type, Object value) { case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] valueT = (Binary[]) value; Binary[] cloneT = new Binary[valueT.length]; System.arraycopy(valueT, 0, cloneT, 0, valueT.length); @@ -872,6 +876,7 @@ private void arrayCopy(Object[] value, int idx, int arrayIndex, int elementIndex case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] arrayT = ((Binary[]) columnValues.get(arrayIndex)); System.arraycopy(value[i], idx, arrayT, elementIndex, remaining); @@ -1174,6 +1179,7 @@ public TsBlock buildTsBlock( case TEXT: case BLOB: case STRING: + case OBJECT: valueBuilder.writeBinary(getBinaryByValueIndex(originRowIndex, columnIndex)); break; default: @@ -1251,6 +1257,7 @@ public int serializedSize() { case TEXT: case BLOB: case STRING: + case OBJECT: for (int rowIdx = 0; rowIdx < rowCount; ++rowIdx) { size += ReadWriteIOUtils.sizeToWrite(getBinaryByValueIndex(rowIdx, columnIndex)); } @@ -1311,6 +1318,7 @@ public void serializeToWAL(IWALByteBufferView buffer) { case TEXT: case BLOB: case STRING: + case OBJECT: Binary valueT = ((Binary[]) columnValues.get(arrayIndex))[elementIndex]; // In some scenario, the Binary in AlignedTVList will be null if this field is empty in // current row. We need to handle this scenario to get rid of NPE. See the similar issue @@ -1391,6 +1399,7 @@ public static AlignedTVList deserialize(DataInputStream stream) throws IOExcepti case TEXT: case BLOB: case STRING: + case OBJECT: Binary[] binaryValues = new Binary[rowCount]; for (int rowIndex = 0; rowIndex < rowCount; ++rowIndex) { binaryValues[rowIndex] = ReadWriteIOUtils.readBinary(stream); @@ -1853,6 +1862,7 @@ public TsPrimitiveType getPrimitiveTypeObject(int rowIndex, int columnIndex) { case TEXT: case BLOB: case STRING: + case OBJECT: return TsPrimitiveType.getByType( TSDataType.TEXT, getBinaryByValueIndex(valueIndex, validColumnIndex)); default: @@ -2089,6 +2099,7 @@ private void writeToColumn( case TEXT: case BLOB: case STRING: + case OBJECT: valueBuilder.writeBinary(getBinaryByValueIndex(originRowIndex, validColumnIndex)); break; default: @@ -2269,6 +2280,7 @@ public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, lo case TEXT: case BLOB: case STRING: + case OBJECT: valueChunkWriter.write( time, isNull ? null : getBinaryByValueIndex(originRowIndex, validColumnIndex), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java index b115713b6eb9..a10018159818 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiAlignedTVListIterator.java @@ -272,6 +272,7 @@ public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, lo break; case TEXT: case BLOB: + case OBJECT: case STRING: valueChunkWriter.write( currentTime, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java index 9f6b6c1b4f63..8e6dd012265e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortMultiTVListIterator.java @@ -165,6 +165,7 @@ public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, lo break; case TEXT: case BLOB: + case OBJECT: case STRING: Binary value = currIterator.getTVList().getBinary(row); chunkWriterImpl.write(time, value); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java index 42de51ac4b72..3a522f5e7146 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiAlignedTVListIterator.java @@ -254,6 +254,7 @@ public TsBlock nextBatch() { case TEXT: case BLOB: case STRING: + case OBJECT: valueBuilder.writeBinary( alignedTVList.getBinaryByValueIndex(valueIndex, validColumnIndex)); break; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java index b735cc7927be..4b3943518399 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/MultiTVListIterator.java @@ -219,6 +219,7 @@ public TsBlock nextBatch() { case TEXT: case BLOB: case STRING: + case OBJECT: while (hasNextTimeValuePair() && builder.getPositionCount() < maxNumberOfPointsInPage) { TVList.TVListIterator iterator = tvListIterators.get(iteratorIndex); Binary binary = iterator.getTVList().getBinary(iterator.getScanOrderIndex(rowIndex)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java index 45aee62f3b02..a1f2842d34a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java @@ -114,6 +114,7 @@ public static TVList newList(TSDataType dataType) { case TEXT: case BLOB: case STRING: + case OBJECT: return BinaryTVList.newList(); case FLOAT: return FloatTVList.newList(); @@ -676,6 +677,7 @@ public static TVList deserialize(DataInputStream stream) throws IOException { switch (dataType) { case TEXT: case BLOB: + case OBJECT: case STRING: return BinaryTVList.deserialize(stream); case FLOAT: @@ -702,6 +704,7 @@ public static TVList deserializeWithoutBitMap(DataInputStream stream) throws IOE case TEXT: case BLOB: case STRING: + case OBJECT: return BinaryTVList.deserializeWithoutBitMap(stream); case FLOAT: return FloatTVList.deserializeWithoutBitMap(stream); @@ -1126,6 +1129,7 @@ && isTimeSatisfied(time)) { case TEXT: case BLOB: case STRING: + case OBJECT: while (index < rows && builder.getPositionCount() < maxNumberOfPointsInPage && paginationController.hasCurLimit()) { @@ -1226,6 +1230,7 @@ public void encodeBatch(IChunkWriter chunkWriter, BatchEncodeInfo encodeInfo, lo case TEXT: case BLOB: case STRING: + case OBJECT: Binary value = getBinary(index); chunkWriterImpl.write(time, value); encodeInfo.dataSizeInChunk += 8L + getBinarySize(value); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java index 0bc72b3205aa..a87fb480d9d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/windowing/window/WindowImpl.java @@ -87,6 +87,7 @@ private void init(EvictableBatchList list, int begin) { case TEXT: case BLOB: case STRING: + case OBJECT: binaryValues = new Binary[size]; for (int i = 0; i < size; ++i) { binaryValues[i] = list.getBinaryByIndex(begin + i); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/function/RecordObjectTypeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/function/RecordObjectTypeTest.java new file mode 100644 index 000000000000..98f2428e8d58 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/function/RecordObjectTypeTest.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.function; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.DiskSpaceInsufficientException; +import org.apache.iotdb.db.queryengine.execution.operator.process.function.partition.Slice; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.RecordIterator; +import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; +import org.apache.iotdb.udf.api.relational.access.Record; +import org.apache.iotdb.udf.api.type.Type; + +import org.apache.tsfile.block.TsBlockBuilderStatus; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilderStatus; +import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.type.ObjectType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Iterator; +import java.util.Optional; + +import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.RecordIterator.OBJECT_ERR_MSG; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class RecordObjectTypeTest { + + private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private File objectDir; + + @Before + public void setup() { + config.setRestrictObjectLimit(true); + try { + objectDir = new File(TierManager.getInstance().getNextFolderForObjectFile()); + } catch (DiskSpaceInsufficientException e) { + throw new RuntimeException(e); + } + } + + @After + public void tearDown() throws IOException { + File[] files = objectDir.listFiles(); + if (files != null) { + for (File file : files) { + Files.delete(file.toPath()); + } + } + config.setRestrictObjectLimit(false); + } + + @Test + public void test() throws IOException { + BinaryColumnBuilder columnBuilder = + new BinaryColumnBuilder(new ColumnBuilderStatus(new TsBlockBuilderStatus(1024)), 1); + columnBuilder.writeBinary(new Binary(createObjectBinary().array())); + RecordIterator recordIterator = + new RecordIterator( + Collections.singletonList(columnBuilder.build()), + Collections.singletonList(ObjectType.OBJECT), + 1); + Slice slice = + new Slice( + 0, + 1, + new Column[] {columnBuilder.build()}, + Collections.singletonList(0), + Collections.emptyList(), + Collections.singletonList(Type.OBJECT)); + testRecordIterator(recordIterator); + testRecordIterator(slice.getRequiredRecordIterator(false)); + } + + private void testRecordIterator(Iterator recordIterator) { + Assert.assertTrue(recordIterator.hasNext()); + Record record = recordIterator.next(); + + Binary result = record.readObject(0); + assertEquals(100, result.getLength()); + for (int j = 0; j < 100; j++) { + assertEquals(j, result.getValues()[j]); + } + + result = record.readObject(0, 10, 2); + Assert.assertArrayEquals(new byte[] {(byte) 10, (byte) 11}, result.getValues()); + + try { + record.getObject(0); + fail("Should throw exception"); + } catch (UnsupportedOperationException e) { + assertEquals(OBJECT_ERR_MSG, e.getMessage()); + } + + try { + record.getBinary(0); + fail("Should throw exception"); + } catch (UnsupportedOperationException e) { + assertEquals(OBJECT_ERR_MSG, e.getMessage()); + } + + Optional objectFile = record.getObjectFile(0); + assertTrue(objectFile.isPresent()); + + assertEquals("(Object) 100 B", record.getString(0)); + Assert.assertFalse(recordIterator.hasNext()); + } + + private ByteBuffer createObjectBinary() throws IOException { + Path testFile1 = Files.createTempFile(objectDir.toPath(), "test_", ".bin"); + byte[] content = new byte[100]; + for (int i = 0; i < 100; i++) { + content[i] = (byte) i; + } + Files.write(testFile1, content); + String relativePath = testFile1.toFile().getName(); + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES + relativePath.length()); + buffer.putLong(100L); + buffer.put(BytesUtils.stringToBytes(relativePath)); + buffer.flip(); + return buffer; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformerTest.java index ffb00505d140..7fbbd6da2fa2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/BlobLengthColumnTransformerTest.java @@ -31,7 +31,7 @@ import java.nio.charset.StandardCharsets; import java.util.Optional; -import static org.apache.tsfile.read.common.type.IntType.INT32; +import static org.apache.tsfile.read.common.type.LongType.INT64; public class BlobLengthColumnTransformerTest { @@ -69,13 +69,13 @@ public void testBlobLengthFromString() { ColumnTransformer childColumnTransformer = mockChildColumnTransformer(binaryColumn); BlobLengthColumnTransformer blobLengthColumnTransformer = - new BlobLengthColumnTransformer(INT32, childColumnTransformer); + new BlobLengthColumnTransformer(INT64, childColumnTransformer); blobLengthColumnTransformer.addReferenceCount(); blobLengthColumnTransformer.evaluate(); Column result = blobLengthColumnTransformer.getColumn(); int expectedLength = input.getBytes(StandardCharsets.UTF_8).length; - Assert.assertEquals(expectedLength, result.getInt(0)); + Assert.assertEquals(expectedLength, result.getLong(0)); } @Test @@ -87,13 +87,13 @@ public void testBlobLengthFromHex() { ColumnTransformer childColumnTransformer = mockChildColumnTransformer(binaryColumn); BlobLengthColumnTransformer blobLengthColumnTransformer = - new BlobLengthColumnTransformer(INT32, childColumnTransformer); + new BlobLengthColumnTransformer(INT64, childColumnTransformer); blobLengthColumnTransformer.addReferenceCount(); blobLengthColumnTransformer.evaluate(); Column result = blobLengthColumnTransformer.getColumn(); int expectedLength = inputBytes.length; - Assert.assertEquals(expectedLength, result.getInt(0)); + Assert.assertEquals(expectedLength, result.getLong(0)); } @Test @@ -109,13 +109,13 @@ public void testBlobLengthMultiRowsWithNull() { ColumnTransformer childColumnTransformer = mockChildColumnTransformer(binaryColumn); BlobLengthColumnTransformer blobLengthColumnTransformer = - new BlobLengthColumnTransformer(INT32, childColumnTransformer); + new BlobLengthColumnTransformer(INT64, childColumnTransformer); blobLengthColumnTransformer.addReferenceCount(); blobLengthColumnTransformer.evaluate(); Column result = blobLengthColumnTransformer.getColumn(); - Assert.assertEquals(inputBytes1.length, result.getInt(0)); + Assert.assertEquals(inputBytes1.length, result.getLong(0)); Assert.assertTrue(result.isNull(1)); - Assert.assertEquals(inputBytes2.length, result.getInt(2)); + Assert.assertEquals(inputBytes2.length, result.getLong(2)); } @Test @@ -133,7 +133,7 @@ public void testBlobLengthWithSelection() { ColumnTransformer child = mockChildColumnTransformer(new BinaryColumn(values.length, Optional.empty(), values)); BlobLengthColumnTransformer blobLengthColumnTransformer = - new BlobLengthColumnTransformer(INT32, child); + new BlobLengthColumnTransformer(INT64, child); blobLengthColumnTransformer.addReferenceCount(); blobLengthColumnTransformer.evaluateWithSelection(booleans); Column result = blobLengthColumnTransformer.getColumn(); @@ -142,7 +142,7 @@ public void testBlobLengthWithSelection() { int expectedValue3 = bytes3.length; Assert.assertTrue(result.isNull(0)); - Assert.assertEquals(expectedValue2, result.getInt(1)); - Assert.assertEquals(expectedValue3, result.getInt(2)); + Assert.assertEquals(expectedValue2, result.getLong(1)); + Assert.assertEquals(expectedValue3, result.getLong(2)); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ObjectTypeFunctionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ObjectTypeFunctionTest.java new file mode 100644 index 000000000000..022257d1ee63 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ObjectTypeFunctionTest.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar; + +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.DiskSpaceInsufficientException; +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; +import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; + +import org.apache.tsfile.block.TsBlockBuilderStatus; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilderStatus; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.read.common.block.column.LongColumn; +import org.apache.tsfile.read.common.type.BlobType; +import org.apache.tsfile.read.common.type.LongType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; + +public class ObjectTypeFunctionTest { + + private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + private File objectDir; + + @Before + public void setup() { + config.setRestrictObjectLimit(true); + try { + objectDir = new File(TierManager.getInstance().getNextFolderForObjectFile()); + } catch (DiskSpaceInsufficientException e) { + throw new RuntimeException(e); + } + } + + @After + public void tearDown() throws IOException { + File[] files = objectDir.listFiles(); + if (files != null) { + for (File file : files) { + Files.delete(file.toPath()); + } + } + config.setRestrictObjectLimit(false); + } + + @Test + public void testLength() throws IOException { + BinaryColumnBuilder columnBuilder = + new BinaryColumnBuilder(new ColumnBuilderStatus(new TsBlockBuilderStatus(1024)), 1); + columnBuilder.writeBinary(new Binary(createObjectBinary().array())); + ColumnTransformer childColumnTransformer = mockChildColumnTransformer(columnBuilder.build()); + ObjectLengthColumnTransformer transformer = + new ObjectLengthColumnTransformer(LongType.INT64, childColumnTransformer); + transformer.addReferenceCount(); + transformer.evaluate(); + Column result = transformer.getColumn(); + Assert.assertTrue(result instanceof LongColumn); + Assert.assertEquals(LongType.INT64, transformer.getType()); + Assert.assertEquals(1, result.getPositionCount()); + for (int i = 0; i < result.getPositionCount(); i++) { + Assert.assertEquals(100, result.getLong(i)); + } + } + + @Test + public void testReadObject1() throws IOException { + BinaryColumnBuilder columnBuilder = + new BinaryColumnBuilder(new ColumnBuilderStatus(new TsBlockBuilderStatus(1024)), 1); + columnBuilder.writeBinary(new Binary(createObjectBinary().array())); + ColumnTransformer childColumnTransformer = mockChildColumnTransformer(columnBuilder.build()); + ReadObjectColumnTransformer transformer = + new ReadObjectColumnTransformer(BlobType.BLOB, childColumnTransformer, Optional.empty()); + transformer.addReferenceCount(); + transformer.evaluate(); + Column result = transformer.getColumn(); + Assert.assertTrue(result instanceof BinaryColumn); + Assert.assertEquals(BlobType.BLOB, transformer.getType()); + Assert.assertEquals(1, result.getPositionCount()); + for (int i = 0; i < result.getPositionCount(); i++) { + Assert.assertEquals(100, result.getBinary(i).getLength()); + for (int j = 0; j < 100; j++) { + Assert.assertEquals(j, result.getBinary(i).getValues()[j]); + } + } + } + + @Test + public void testReadObject2() throws IOException { + BinaryColumnBuilder columnBuilder = + new BinaryColumnBuilder(new ColumnBuilderStatus(new TsBlockBuilderStatus(1024)), 1); + columnBuilder.writeBinary(new Binary(createObjectBinary().array())); + ColumnTransformer childColumnTransformer = mockChildColumnTransformer(columnBuilder.build()); + ReadObjectColumnTransformer transformer = + new ReadObjectColumnTransformer( + BlobType.BLOB, 10, childColumnTransformer, Optional.empty()); + transformer.addReferenceCount(); + transformer.evaluate(); + Column result = transformer.getColumn(); + Assert.assertTrue(result instanceof BinaryColumn); + Assert.assertEquals(BlobType.BLOB, transformer.getType()); + Assert.assertEquals(1, result.getPositionCount()); + for (int i = 0; i < result.getPositionCount(); i++) { + Assert.assertEquals(90, result.getBinary(i).getLength()); + for (int j = 10; j < 100; j++) { + Assert.assertEquals(j, result.getBinary(i).getValues()[j - 10]); + } + } + } + + @Test + public void testReadObject3() throws IOException { + BinaryColumnBuilder columnBuilder = + new BinaryColumnBuilder(new ColumnBuilderStatus(new TsBlockBuilderStatus(1024)), 1); + columnBuilder.writeBinary(new Binary(createObjectBinary().array())); + ColumnTransformer childColumnTransformer = mockChildColumnTransformer(columnBuilder.build()); + ReadObjectColumnTransformer transformer = + new ReadObjectColumnTransformer( + BlobType.BLOB, 10, 2, childColumnTransformer, Optional.empty()); + transformer.addReferenceCount(); + transformer.evaluate(); + Column result = transformer.getColumn(); + Assert.assertTrue(result instanceof BinaryColumn); + Assert.assertEquals(BlobType.BLOB, transformer.getType()); + Assert.assertEquals(1, result.getPositionCount()); + for (int i = 0; i < result.getPositionCount(); i++) { + Assert.assertEquals(2, result.getBinary(i).getLength()); + Assert.assertArrayEquals(new byte[] {(byte) 10, (byte) 11}, result.getBinary(i).getValues()); + } + } + + private ByteBuffer createObjectBinary() throws IOException { + Path testFile1 = Files.createTempFile(objectDir.toPath(), "test_", ".bin"); + byte[] content = new byte[100]; + for (int i = 0; i < 100; i++) { + content[i] = (byte) i; + } + Files.write(testFile1, content); + String relativePath = testFile1.toFile().getName(); + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES + relativePath.length()); + buffer.putLong(100L); + buffer.put(BytesUtils.stringToBytes(relativePath)); + buffer.flip(); + return buffer; + } + + private ColumnTransformer mockChildColumnTransformer(Column column) { + ColumnTransformer mockColumnTransformer = Mockito.mock(ColumnTransformer.class); + Mockito.when(mockColumnTransformer.getColumn()).thenReturn(column); + Mockito.doNothing().when(mockColumnTransformer).tryEvaluate(); + Mockito.doNothing().when(mockColumnTransformer).clearCache(); + Mockito.doNothing().when(mockColumnTransformer).evaluateWithSelection(Mockito.any()); + return mockColumnTransformer; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/object/ObjectTypeCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/object/ObjectTypeCompactionTest.java new file mode 100644 index 000000000000..4fd7e8c432d9 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/object/ObjectTypeCompactionTest.java @@ -0,0 +1,314 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.compaction.object; + +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; +import org.apache.iotdb.commons.schema.table.column.TagColumnSchema; +import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.DiskSpaceInsufficientException; +import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; +import org.apache.iotdb.db.storageengine.dataregion.Base32ObjectPath; +import org.apache.iotdb.db.storageengine.dataregion.IObjectPath; +import org.apache.iotdb.db.storageengine.dataregion.PlainObjectPath; +import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.SettleCompactionTask; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; +import org.apache.iotdb.db.utils.ObjectTypeUtils; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.ColumnSchema; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.StringArrayDeviceID; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; + +public class ObjectTypeCompactionTest extends AbstractCompactionTest { + + private static final TableSchema tableSchema = + new TableSchema( + "t1", + Arrays.asList( + new ColumnSchema("device", TSDataType.STRING, ColumnCategory.TAG), + new ColumnSchema("s1", TSDataType.OBJECT, ColumnCategory.FIELD))); + + private String threadName; + private File objectDir; + private File regionDir; + + private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + @Before + @Override + public void setUp() + throws IOException, WriteProcessException, MetadataException, InterruptedException { + config.setRestrictObjectLimit(true); + this.threadName = Thread.currentThread().getName(); + Thread.currentThread().setName("pool-1-IoTDB-Compaction-Worker-1"); + DataNodeTableCache.getInstance().invalid(this.COMPACTION_TEST_SG); + createTable("t1", 1); + super.setUp(); + try { + objectDir = new File(TierManager.getInstance().getNextFolderForObjectFile()); + regionDir = new File(objectDir, "0"); + regionDir.mkdirs(); + } catch (DiskSpaceInsufficientException e) { + throw new RuntimeException(e); + } + } + + @After + @Override + public void tearDown() throws IOException, StorageEngineException { + super.tearDown(); + Thread.currentThread().setName(threadName); + DataNodeTableCache.getInstance().invalid(this.COMPACTION_TEST_SG); + File[] files = objectDir.listFiles(); + if (files != null) { + for (File file : files) { + FileUtils.deleteFileOrDirectory(file); + } + } + config.setRestrictObjectLimit(false); + } + + public void createTable(String tableName, long ttl) { + TsTable tsTable = new TsTable(tableName); + tsTable.addColumnSchema(new TagColumnSchema("device", TSDataType.STRING)); + tsTable.addColumnSchema( + new FieldColumnSchema("s1", TSDataType.OBJECT, TSEncoding.PLAIN, CompressionType.LZ4)); + tsTable.addProp(TsTable.TTL_PROPERTY, ttl + ""); + DataNodeTableCache.getInstance().preUpdateTable(this.COMPACTION_TEST_SG, tsTable, null); + DataNodeTableCache.getInstance().commitUpdateTable(this.COMPACTION_TEST_SG, tableName, null); + } + + @Test + public void testSeqCompactionWithTTL() throws IOException, WriteProcessException { + Pair pair1 = + generateTsFileAndObject(true, System.currentTimeMillis() - 10000, 0); + Pair pair2 = + generateTsFileAndObject(true, System.currentTimeMillis() + 1000000, 100); + tsFileManager.add(pair1.getLeft(), true); + tsFileManager.add(pair2.getLeft(), true); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + 0, + tsFileManager, + tsFileManager.getTsFileList(true), + true, + new ReadChunkCompactionPerformer(), + 0); + Assert.assertTrue(task.start()); + Assert.assertFalse(pair1.getRight().exists()); + Assert.assertTrue(pair2.getRight().exists()); + } + + @Test + public void testUnseqCompactionWithTTL() throws IOException, WriteProcessException { + Pair pair1 = + generateTsFileAndObject(false, System.currentTimeMillis() + 100000, 1); + Pair pair2 = + generateTsFileAndObject(false, System.currentTimeMillis() - 1000000, 0); + tsFileManager.add(pair1.getLeft(), false); + tsFileManager.add(pair2.getLeft(), false); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + 0, + tsFileManager, + tsFileManager.getTsFileList(false), + false, + new FastCompactionPerformer(false), + 0); + Assert.assertTrue(task.start()); + Assert.assertFalse(pair2.getRight().exists()); + Assert.assertTrue(pair1.getRight().exists()); + } + + @Test + public void testUnseqCompactionWithReadPointWithTTL() throws IOException, WriteProcessException { + Pair pair1 = + generateTsFileAndObject(false, System.currentTimeMillis() + 100000, 0); + Pair pair2 = + generateTsFileAndObject(false, System.currentTimeMillis() - 1000000, 0); + tsFileManager.add(pair1.getLeft(), false); + tsFileManager.add(pair2.getLeft(), false); + InnerSpaceCompactionTask task = + new InnerSpaceCompactionTask( + 0, + tsFileManager, + tsFileManager.getTsFileList(false), + false, + new ReadPointCompactionPerformer(), + 0); + Assert.assertTrue(task.start()); + Assert.assertTrue(pair1.getRight().exists()); + Assert.assertFalse(pair2.getRight().exists()); + } + + @Test + public void testCrossCompactionWithTTL() throws IOException, WriteProcessException { + Pair pair1 = + generateTsFileAndObject(true, System.currentTimeMillis() + 100000, 1); + Pair pair2 = + generateTsFileAndObject(false, System.currentTimeMillis() - 1000000, 2); + tsFileManager.add(pair1.getLeft(), true); + tsFileManager.add(pair2.getLeft(), false); + CrossSpaceCompactionTask task = + new CrossSpaceCompactionTask( + 0, + tsFileManager, + tsFileManager.getTsFileList(true), + tsFileManager.getTsFileList(false), + new FastCompactionPerformer(true), + 1, + 0); + Assert.assertTrue(task.start()); + Assert.assertFalse(pair2.getRight().exists()); + Assert.assertTrue(pair1.getRight().exists()); + } + + @Test + public void testSettleCompaction() throws IOException, WriteProcessException { + Pair pair1 = + generateTsFileAndObject(true, System.currentTimeMillis() - 10000, 3); + Pair pair2 = + generateTsFileAndObject(true, System.currentTimeMillis() + 1000000, 0); + tsFileManager.add(pair1.getLeft(), true); + tsFileManager.add(pair2.getLeft(), true); + SettleCompactionTask task = + new SettleCompactionTask( + 0, + tsFileManager, + tsFileManager.getTsFileList(true), + Collections.emptyList(), + true, + new FastCompactionPerformer(true), + 0); + Assert.assertTrue(task.start()); + Assert.assertFalse(pair1.getRight().exists()); + Assert.assertTrue(pair2.getRight().exists()); + } + + @Test + public void testPlainObjectBinaryReplaceRegionId() { + IObjectPath objectPath = new PlainObjectPath(1, 0, new StringArrayDeviceID("t1.d1"), "s1"); + ByteBuffer buffer = + ByteBuffer.allocate(Long.BYTES + objectPath.getSerializeSizeToObjectValue()); + buffer.putLong(10); + objectPath.serializeToObjectValue(buffer); + + Binary origin = new Binary(buffer.array()); + Binary result = ObjectTypeUtils.replaceRegionIdForObjectBinary(10, origin); + ByteBuffer deserializeBuffer = ByteBuffer.wrap(result.getValues()); + deserializeBuffer.getLong(); + Assert.assertEquals( + new PlainObjectPath(10, 0, new StringArrayDeviceID("t1.d1"), "s1").toString(), + IObjectPath.getDeserializer().deserializeFromObjectValue(deserializeBuffer).toString()); + } + + @Test + public void testBase32ObjectBinaryReplaceRegionId() { + config.setRestrictObjectLimit(false); + try { + IObjectPath objectPath = new Base32ObjectPath(1, 0, new StringArrayDeviceID("t1.d1"), "s1"); + ByteBuffer buffer = + ByteBuffer.allocate(Long.BYTES + objectPath.getSerializeSizeToObjectValue()); + buffer.putLong(10); + objectPath.serializeToObjectValue(buffer); + + Binary origin = new Binary(buffer.array()); + Binary result = ObjectTypeUtils.replaceRegionIdForObjectBinary(10, origin); + ByteBuffer deserializeBuffer = ByteBuffer.wrap(result.getValues()); + deserializeBuffer.getLong(); + Assert.assertEquals( + new Base32ObjectPath(10, 0, new StringArrayDeviceID("t1.d1"), "s1").toString(), + IObjectPath.getDeserializer().deserializeFromObjectValue(deserializeBuffer).toString()); + } finally { + config.setRestrictObjectLimit(true); + } + } + + private Pair generateTsFileAndObject( + boolean seq, long timestamp, int regionIdInTsFile) throws IOException, WriteProcessException { + TsFileResource resource = createEmptyFileAndResource(seq); + Path testFile1 = Files.createTempFile(regionDir.toPath(), "test_", ".bin"); + byte[] content = new byte[100]; + for (int i = 0; i < 100; i++) { + content[i] = (byte) i; + } + Files.write(testFile1, content); + String relativePathInTsFile = regionIdInTsFile + File.separator + testFile1.toFile().getName(); + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES + relativePathInTsFile.length()); + buffer.putLong(100L); + buffer.put(BytesUtils.stringToBytes(relativePathInTsFile)); + buffer.flip(); + IDeviceID deviceID = new StringArrayDeviceID("t1", "d1"); + try (TsFileIOWriter writer = new TsFileIOWriter(resource.getTsFile())) { + writer.getSchema().registerTableSchema(tableSchema); + writer.startChunkGroup(deviceID); + AlignedChunkWriterImpl alignedChunkWriter = + new AlignedChunkWriterImpl(Arrays.asList(new MeasurementSchema("s1", TSDataType.OBJECT))); + alignedChunkWriter.write(timestamp); + alignedChunkWriter.write(timestamp, new Binary(buffer.array()), false); + alignedChunkWriter.sealCurrentPage(); + alignedChunkWriter.writeToFileWriter(writer); + writer.endChunkGroup(); + writer.endFile(); + } + resource.updateStartTime(deviceID, 1); + resource.updateEndTime(deviceID, 1); + resource.serialize(); + resource.deserialize(); + resource.setStatus(TsFileResourceStatus.NORMAL); + return new Pair<>(resource, testFile1.toFile()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/PrimitiveArrayManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/PrimitiveArrayManagerTest.java index 3ab79997baf8..d5c7b3e865f9 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/PrimitiveArrayManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/datastructure/PrimitiveArrayManagerTest.java @@ -92,6 +92,7 @@ public void testUpdateLimits() { break; case TEXT: case BLOB: + case OBJECT: case STRING: Assert.assertTrue(o instanceof Binary[]); Assert.assertEquals(ARRAY_SIZE, ((Binary[]) o).length); diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 6bbc4124a9fb..4330a11b1f60 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1302,6 +1302,16 @@ enable_tsfile_validation=false # Unit: ms tier_ttl_in_ms=-1 +# The maximum size limit for a single object file. +# effectiveMode: hot_reload +# Datatype: long +max_object_file_size_in_byte=4294967296 + +# There are no special restrictions on table names, column names, and device names of the OBJECT type. +# effectiveMode: first_start +# Datatype: boolean +restrict_object_limit=false + #################### ### Compaction Configurations #################### diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java index 13fbf23972f0..adf72842797e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java @@ -248,6 +248,7 @@ private IoTDBConstant() {} public static final String DATA_FOLDER_NAME = "data"; public static final String SEQUENCE_FOLDER_NAME = "sequence"; public static final String UNSEQUENCE_FOLDER_NAME = "unsequence"; + public static final String OBJECT_FOLDER_NAME = "object"; public static final String FILE_NAME_SEPARATOR = "-"; public static final String CONSENSUS_FOLDER_NAME = "consensus"; public static final String DATA_REGION_FOLDER_NAME = "data_region"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/ObjectFileNotExist.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/ObjectFileNotExist.java new file mode 100644 index 000000000000..05add08b218f --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/exception/ObjectFileNotExist.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.exception; + +import static org.apache.iotdb.rpc.TSStatusCode.OBJECT_NOT_EXISTS; + +public class ObjectFileNotExist extends IoTDBRuntimeException { + + private static final String ERROR_MSG = "Object file %s does not exist"; + + public ObjectFileNotExist(String relativeObjectPath) { + super(String.format(ERROR_MSG, relativeObjectPath), OBJECT_NOT_EXISTS.getStatusCode()); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 6c3d85c12aa6..8336f0f4aa02 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -20,6 +20,7 @@ package org.apache.iotdb.commons.schema.table; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.exception.runtime.SchemaExecutionException; import org.apache.iotdb.commons.schema.table.column.AttributeColumnSchema; import org.apache.iotdb.commons.schema.table.column.FieldColumnSchema; @@ -29,6 +30,7 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; +import org.apache.iotdb.rpc.TSStatusCode; import com.google.common.collect.ImmutableList; import org.apache.tsfile.enums.TSDataType; @@ -69,6 +71,8 @@ public class TsTable { public static final String TTL_PROPERTY = "ttl"; public static final Set TABLE_ALLOWED_PROPERTIES = Collections.singleton(TTL_PROPERTY); + private static final String OBJECT_STRING_ERROR = + "When there are object fields, the %s %s shall not be '.', '..' or contain './', '.\\'"; protected String tableName; private final Map columnSchemaMap = new LinkedHashMap<>(); @@ -410,6 +414,33 @@ public void setProps(Map props) { executeWrite(() -> this.props = props); } + public void checkTableNameAndObjectNames4Object() throws MetadataException { + if (isInvalid4ObjectType(tableName)) { + throw new MetadataException( + getObjectStringError("tableName", tableName), + TSStatusCode.SEMANTIC_ERROR.getStatusCode()); + } + for (final TsTableColumnSchema schema : columnSchemaMap.values()) { + if (schema.getDataType().equals(TSDataType.OBJECT) + && isInvalid4ObjectType(schema.getColumnName())) { + throw new MetadataException( + getObjectStringError("objectName", schema.getColumnName()), + TSStatusCode.SEMANTIC_ERROR.getStatusCode()); + } + } + } + + public static boolean isInvalid4ObjectType(final String column) { + return column.equals(".") + || column.equals("..") + || column.contains("./") + || column.contains(".\\"); + } + + public static String getObjectStringError(final String columnType, final String columnName) { + return String.format(OBJECT_STRING_ERROR, columnType, columnName); + } + @Override public boolean equals(Object o) { return super.equals(o); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java index be0d16807e21..c3fe93420660 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFAbs.java @@ -64,6 +64,7 @@ public void transform(Row row, PointCollector collector) collector.putDouble(time, Math.abs(row.getDouble(0))); break; case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case TEXT: @@ -103,6 +104,7 @@ public Object transform(Row row) throws IOException { case TIMESTAMP: case STRING: case BLOB: + case OBJECT: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( @@ -131,6 +133,7 @@ public void transform(Column[] columns, ColumnBuilder builder) throws Exception transformDouble(columns, builder); return; case BLOB: + case OBJECT: case STRING: case TEXT: case TIMESTAMP: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java index bb12f7ff4a58..045e49b064b7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFBottomK.java @@ -73,6 +73,7 @@ protected void constructPQ() throws UDFInputSeriesDataTypeNotValidException { }); break; case BLOB: + case OBJECT: case BOOLEAN: default: // This will not happen. diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java index 00b3fd475501..cb5cbcc0486b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonDerivative.java @@ -61,6 +61,7 @@ protected void doTransform(Row row, PointCollector collector) case TEXT: case STRING: case BLOB: + case OBJECT: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java index 75305f731651..7f8d81f6b829 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFCommonValueDifference.java @@ -56,6 +56,7 @@ protected void doTransform(Row row, PointCollector collector) break; case STRING: case BLOB: + case OBJECT: case TIMESTAMP: case TEXT: case BOOLEAN: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java index 209802c26cb9..f3436859f2a4 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFConst.java @@ -57,6 +57,7 @@ public class UDTFConst implements UDTF { VALID_TYPES.add(TSDataType.TEXT.name()); VALID_TYPES.add(TSDataType.STRING.name()); VALID_TYPES.add(TSDataType.BLOB.name()); + VALID_TYPES.add(TSDataType.OBJECT.name()); } private TSDataType dataType; @@ -107,6 +108,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati binaryValue = BytesUtils.valueOf(parameters.getString("value")); break; case BLOB: + case OBJECT: binaryValue = new Binary(BlobUtils.parseBlobString(parameters.getString("value"))); break; default: @@ -141,6 +143,7 @@ public void transform(Row row, PointCollector collector) throws Exception { case TEXT: case STRING: case BLOB: + case OBJECT: collector.putBinary(row.getTime(), UDFBinaryTransformer.transformToUDFBinary(binaryValue)); break; default: @@ -166,6 +169,7 @@ public Object transform(Row row) throws IOException { case TEXT: case STRING: case BLOB: + case OBJECT: return UDFBinaryTransformer.transformToUDFBinary(binaryValue); default: throw new UnsupportedOperationException(); @@ -257,6 +261,7 @@ public void transform(Column[] columns, ColumnBuilder builder) throws Exception case TEXT: case STRING: case BLOB: + case OBJECT: for (int i = 0; i < count; i++) { boolean hasWritten = false; for (int j = 0; j < columns.length - 1; j++) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java index b76b0276e7cf..de6232ac6db3 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFContinuouslySatisfy.java @@ -103,6 +103,7 @@ public void transform(Row row, PointCollector collector) case TEXT: case STRING: case BLOB: + case OBJECT: case TIMESTAMP: case DATE: default: @@ -230,6 +231,7 @@ public void terminate(PointCollector collector) case DATE: case STRING: case BLOB: + case OBJECT: case TEXT: default: // This will not happen. diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java index d01673276137..21506698605f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketAggSample.java @@ -482,6 +482,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) aggregator.aggregateDouble(rowWindow, collector); break; case BLOB: + case OBJECT: case TEXT: case DATE: case STRING: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java index cf80380eea2a..fea5cb694fa7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketM4Sample.java @@ -64,6 +64,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) case STRING: case TEXT: case BLOB: + case OBJECT: default: // This will not happen throw new UDFInputSeriesDataTypeNotValidException( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java index f3b3aacba378..85490ac534e4 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketOutlierSample.java @@ -658,6 +658,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) break; case TEXT: case BLOB: + case OBJECT: case DATE: case STRING: case BOOLEAN: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java index f77b7ab8a729..566e02e7b58f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFEqualSizeBucketRandomSample.java @@ -66,6 +66,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) case DATE: case STRING: case BLOB: + case OBJECT: case TEXT: default: // This will not happen diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java index f24b9d6ae7b8..4c17a2706973 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFInRange.java @@ -86,6 +86,7 @@ public void transform(Row row, PointCollector collector) collector.putBoolean(time, row.getDouble(0) >= lower && upper >= row.getDouble(0)); break; case BLOB: + case OBJECT: case TEXT: case DATE: case STRING: @@ -123,6 +124,7 @@ public Object transform(Row row) throws Exception { case STRING: case TEXT: case BLOB: + case OBJECT: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( @@ -151,6 +153,7 @@ public void transform(Column[] columns, ColumnBuilder builder) throws Exception transformDouble(columns, builder); return; case BLOB: + case OBJECT: case TEXT: case DATE: case STRING: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFJexl.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFJexl.java index d554cfe25720..0c5ccd73b492 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFJexl.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFJexl.java @@ -98,6 +98,7 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati case TIMESTAMP: case DATE: case BLOB: + case OBJECT: default: throw new UDFInputSeriesDataTypeNotValidException( 0, @@ -170,6 +171,7 @@ public void transform(Row row, PointCollector collector) case DATE: case STRING: case BLOB: + case OBJECT: case INT64: case INT32: case FLOAT: @@ -346,6 +348,7 @@ public void getValues(Row row) throws IOException, UDFInputSeriesDataTypeNotVali break; case STRING: case BLOB: + case OBJECT: case DATE: case TIMESTAMP: default: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java index 824740b8c9b0..3de46df72a06 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFM4.java @@ -138,6 +138,7 @@ public void transform(RowWindow rowWindow, PointCollector collector) transformDouble(rowWindow, collector); break; case BLOB: + case OBJECT: case DATE: case STRING: case TIMESTAMP: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java index de4f8edcec5d..947cbdb2c013 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFMath.java @@ -91,6 +91,7 @@ public void transform(Row row, PointCollector collector) case TIMESTAMP: case DATE: case BLOB: + case OBJECT: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( @@ -119,6 +120,7 @@ public Object transform(Row row) throws IOException { return transformer.transform(row.getDouble(0)); case DATE: case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case TEXT: @@ -155,6 +157,7 @@ public void transform(Column[] columns, ColumnBuilder builder) throws Exception case STRING: case TIMESTAMP: case BLOB: + case OBJECT: case DATE: default: // This will not happen. diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java index 6812d2d2d8ee..077db957f6e2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeDerivative.java @@ -57,6 +57,7 @@ protected void doTransform(Row row, PointCollector collector) break; case DATE: case BLOB: + case OBJECT: case STRING: case TIMESTAMP: case BOOLEAN: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java index 1183e5397fef..fcb68817f153 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFNonNegativeValueDifference.java @@ -59,6 +59,7 @@ protected void doTransform(Row row, PointCollector collector) case STRING: case TIMESTAMP: case BLOB: + case OBJECT: case DATE: default: // This will not happen. diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java index 66b5cd1add8e..b0c95bd0d23b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFOnOff.java @@ -145,6 +145,7 @@ public void transform(Column[] columns, ColumnBuilder builder) throws Exception transformDouble(columns, builder); return; case BLOB: + case OBJECT: case DATE: case STRING: case TIMESTAMP: diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java index 49faa398f1a7..a888c998d8d8 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFSelectK.java @@ -107,6 +107,7 @@ public void transform(Row row, PointCollector collector) transformString(row.getTime(), row.getString(0)); break; case BLOB: + case OBJECT: case BOOLEAN: default: // This will not happen. @@ -180,6 +181,7 @@ public void terminate(PointCollector collector) } break; case BLOB: + case OBJECT: case BOOLEAN: default: // This will not happen. diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java index 1838b85022b3..b8d94308ed96 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFTopK.java @@ -53,6 +53,7 @@ protected void constructPQ() throws UDFInputSeriesDataTypeNotValidException { break; case BOOLEAN: case BLOB: + case OBJECT: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java index c99002193294..d4487fdfd953 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/UDTFValueTrend.java @@ -71,6 +71,7 @@ protected void updatePreviousValue(Row row) case STRING: case DATE: case BLOB: + case OBJECT: default: // This will not happen. throw new UDFInputSeriesDataTypeNotValidException( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java index e20ca4dd0737..5d22ff58d813 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/builtin/relational/TableBuiltinScalarFunction.java @@ -111,6 +111,7 @@ public enum TableBuiltinScalarFunction { SPOOKY_HASH_V2_64("spooky_hash_v2_64"), LPAD("lpad"), RPAD("rpad"), + READ_OBJECT("read_object"), ; private final String functionName; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java index 83792ebf66af..cf9ecf89f96c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/udf/utils/UDFDataTypeTransformer.java @@ -30,6 +30,7 @@ import org.apache.tsfile.read.common.type.FloatType; import org.apache.tsfile.read.common.type.IntType; import org.apache.tsfile.read.common.type.LongType; +import org.apache.tsfile.read.common.type.ObjectType; import org.apache.tsfile.read.common.type.StringType; import org.apache.tsfile.read.common.type.TimestampType; @@ -82,6 +83,8 @@ public static Type transformReadTypeToUDFDataType(org.apache.tsfile.read.common. return Type.BLOB; case STRING: return Type.STRING; + case OBJECT: + return Type.OBJECT; default: throw new IllegalArgumentException("Invalid input: " + type); } @@ -112,6 +115,8 @@ public static org.apache.tsfile.read.common.type.Type transformUDFDataTypeToRead return BlobType.BLOB; case STRING: return StringType.STRING; + case OBJECT: + return ObjectType.OBJECT; default: throw new IllegalArgumentException("Invalid input: " + type); } @@ -139,6 +144,8 @@ private static Type getUDFDataType(byte type) { return Type.BLOB; case 11: return Type.STRING; + case 12: + return Type.OBJECT; default: throw new IllegalArgumentException("Invalid input: " + type); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java index d0b57328dcb6..bbc3a101edbb 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/SerializeUtils.java @@ -174,6 +174,7 @@ public static BatchData deserializeBatchData(ByteBuffer buffer) { } break; case BLOB: + case OBJECT: case STRING: case TEXT: for (int i = 0; i < length; i++) { @@ -218,6 +219,7 @@ public static BatchData deserializeBatchData(ByteBuffer buffer) { values[j] = new TsPrimitiveType.TsFloat(buffer.getFloat()); break; case BLOB: + case OBJECT: case STRING: case TEXT: int len = buffer.getInt(); @@ -315,6 +317,7 @@ public static void serializeTVPairs( dataOutputStream.writeInt(timeValuePairs.size()); switch (timeValuePairs.get(0).getValue().getDataType()) { case BLOB: + case OBJECT: case STRING: case TEXT: serializeTextTVPairs(timeValuePairs, dataOutputStream); @@ -355,6 +358,7 @@ public static void serializeTVPair( switch (dataType) { case STRING: case BLOB: + case OBJECT: case TEXT: dataOutputStream.writeLong(timeValuePair.getTimestamp()); if (timeValuePair.getTimestamp() != Long.MIN_VALUE) { @@ -506,6 +510,7 @@ public static List deserializeTVPairs(ByteBuffer buffer) { deserializeBooleanTVPairs(buffer, ret, size, dataType); break; case BLOB: + case OBJECT: case STRING: case TEXT: deserializeTextTVPairs(buffer, ret, size, dataType); @@ -539,6 +544,7 @@ public static TimeValuePair deserializeTVPair(ByteBuffer buffer) { case BOOLEAN: return new TimeValuePair(time, TsPrimitiveType.getByType(dataType, buffer.get() == 1)); case BLOB: + case OBJECT: case STRING: case TEXT: int bytesLen = buffer.getInt(); diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index f2b8ec6b8b07..01bde7c378dc 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -50,6 +50,7 @@ struct TGlobalConfig { 11: optional i32 tagAttributeTotalSize 12: optional bool isEnterprise 13: optional i64 timePartitionOrigin + 14: optional bool restrictObjectLimit } struct TRatisConfig { diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift index eb0a3430899e..48afb89d3366 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift @@ -669,5 +669,5 @@ service IClientRPCService { TSConnectionInfoResp fetchAllConnectionsInfo(); /** For other node's call */ - common.TSStatus testConnectionEmptyRPC() + common.TSStatus testConnectionEmptyRPC(); } \ No newline at end of file diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index caaf44c16a77..469d2bb006ae 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -773,6 +773,12 @@ struct TKillQueryInstanceReq { 2: optional string allowedUsername } +struct TReadObjectReq { + 1: string relativePath + 2: i64 offset + 3: i32 size +} + /** * END: Used for EXPLAIN ANALYZE **/ @@ -1257,6 +1263,8 @@ service IDataNodeRPCService { * Write an audit log entry to the DataNode's AuditEventLogger */ common.TSStatus writeAuditLog(TAuditLogReq req); + + binary readObject(TReadObjectReq req); } service MPPDataExchangeService { diff --git a/pom.xml b/pom.xml index 4ba65896c97f..66c7116a9fb5 100644 --- a/pom.xml +++ b/pom.xml @@ -173,7 +173,7 @@ 0.14.1 1.9 1.5.6-3 - 2.2.0-251113-SNAPSHOT + 2.2.0-251210-SNAPSHOT