diff --git a/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java b/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java index 64a5d178226a..cdb884dfc1b2 100644 --- a/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java +++ b/example/influxdb-protocol-example/src/main/java/org/apache/iotdb/influxdb/InfluxDBExample.java @@ -102,7 +102,7 @@ private static void queryData() { query = new Query( - "select count(temperature),first(temperature),last(temperature),max(temperature),mean(temperature),median(temperature),min(temperature),mode(temperature),spread(temperature),stddev(temperature),sum(temperature) from student where ((workshop=\"A1\" and production=\"B1\" and cell =\"C1\" ) or temperature< 15 )", + "select count(temperature),first(temperature),last(temperature),max(temperature),mean(temperature),median(temperature),min(temperature),mode(temperature),spread(temperature),stddev(temperature),sum(temperature) from factory where ((workshop=\"A1\" and production=\"B1\" and cell =\"C1\" ) or temperature< 15 )", database); result = influxDB.query(query); System.out.println("query2 result:" + result.getResults().get(0).getSeries().get(0).toString()); diff --git a/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java b/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java index d8c97dc2b4ab..047cada0f145 100644 --- a/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java +++ b/influxdb-protocol/src/test/java/org/apache/iotdb/influxdb/integration/IoTDBInfluxDBIT.java @@ -33,11 +33,15 @@ import org.testcontainers.containers.GenericContainer; import org.testcontainers.utility.DockerImageName; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class IoTDBInfluxDBIT { @@ -164,8 +168,9 @@ public void testCommonQueryColumn() { QueryResult.Series series = result.getResults().get(0).getSeries().get(0); String[] retArray = new String[] {"time", "name", "sex", "province", "country", "score", "tel"}; + Set columnNames = new HashSet<>(Arrays.asList(retArray)); for (int i = 0; i < series.getColumns().size(); i++) { - assertEquals(retArray[i], series.getColumns().get(i)); + assertTrue(columnNames.contains(series.getColumns().get(i))); } } diff --git a/pom.xml b/pom.xml index 44f42d20ad10..192c89a9512a 100644 --- a/pom.xml +++ b/pom.xml @@ -121,6 +121,7 @@ trigger-api rewrite-tsfile-tool external-api + schema-engine-tag diff --git a/schema-engine-tag/README.md b/schema-engine-tag/README.md new file mode 100644 index 000000000000..e4d70f9e1502 --- /dev/null +++ b/schema-engine-tag/README.md @@ -0,0 +1,190 @@ + + +# Tag Schema Region +`TagSchemaRegion` is an implementation of `SchemaRegion` +
+  _____             _____      _                           ______           _             
+|_   _|           /  ___|    | |                          | ___ \         (_)            
+  | | __ _  __ _  \ `--.  ___| |__   ___ _ __ ___   __ _  | |_/ /___  __ _ _  ___  _ __  
+  | |/ _` |/ _` |  `--. \/ __| '_ \ / _ \ '_ ` _ \ / _` | |    // _ \/ _` | |/ _ \| '_ \ 
+  | | (_| | (_| | /\__/ / (__| | | |  __/ | | | | | (_| | | |\ \  __/ (_| | | (_) | | | |
+  \_/\__,_|\__, | \____/ \___|_| |_|\___|_| |_| |_|\__,_| \_| \_\___|\__, |_|\___/|_| |_|
+            __/ |                                                     __/ |              
+           |___/                                                     |___/ > version 0.14.0-SNAPSHOT
+
+ +# How To Use + +Firstly, you should package **schema-engine-tag** by the following command: + +```shell +mvn clean package -pl schema-engine-tag -am -DskipTests +``` + +After that, you can get a **conf** directory and a **lib** directory in +schema-engine-tag/target/schema-engine-tag. Copy the file in the conf directory to the conf directory of server, +and copy the files in the lib directory to the lib directory of server. + +Then, open the **iotdb-datanode.properties** in the conf directory of server, and set the `schema_engine_mode` to +**Tag**, set the `enable_id_table` to **true**. Restart the IoTDB, the system will use `TagSchemaRegion` to manage +the metadata. + +## Use Cli + +IoTDB offers different ways to interact with server, here we introduce the basic steps of using Cli tool to insert and query data. +The command line cli is interactive, so you should see the welcome logo and statements if everything is ready: +```sql +--------------------- +Starting IoTDB Cli +--------------------- + _____ _________ ______ ______ +|_ _| | _ _ ||_ _ `.|_ _ \ + | | .--.|_/ | | \_| | | `. \ | |_) | + | | / .'`\ \ | | | | | | | __'. + _| |_| \__. | _| |_ _| |_.' /_| |__) | +|_____|'.__.' |_____| |______.'|_______/ version 0.14.0-SNAPSHOT + + +IoTDB> login successfully +``` +### create timeseries + +- create timeseries + +```sql +IoTDB> create timeseries root.ln.tag1.a.tag2.b.status with datatype=BOOLEAN,encoding=PLAIN +Msg: The statement is executed successfully. +``` +- create aligned timeseries + +```sql +IoTDB> CREATE ALIGNED TIMESERIES root.ln.tag1.a.tag2.c(latitude FLOAT encoding=PLAIN compressor=SNAPPY, longitude FLOAT encoding=PLAIN compressor=SNAPPY) + +Msg: The statement is executed successfully. +``` + +### show timeserie + +- point query + +enter a full path + +```sql +IoTDB> show timeseries root.ln.tag2.c.tag1.a ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| root.ln.tag1.a.tag2.c.latitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| +|root.ln.tag1.a.tag2.c.longitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +``` + +- batch query + +paths ending in ".**" indicate batch query + +```sql +IoTDB> show timeseries root.ln.tag1.a.** ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| root.ln.tag1.a.tag2.b.status| null| root.ln| BOOLEAN| PLAIN| SNAPPY|null| null| +| root.ln.tag1.a.tag2.c.latitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| +|root.ln.tag1.a.tag2.c.longitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ + +IoTDB> show timeseries root.ln.tag2.c.** ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| root.ln.tag1.a.tag2.c.latitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| +|root.ln.tag1.a.tag2.c.longitude| null| root.ln| FLOAT| PLAIN| SNAPPY|null| null| ++-------------------------------+-----+-------------+--------+--------+-----------+----+----------+ + +IoTDB> show timeseries root.ln.tag2.b.** ++----------------------------+-----+-------------+--------+--------+-----------+----+----------+ +| timeseries|alias|storage group|dataType|encoding|compression|tags|attributes| ++----------------------------+-----+-------------+--------+--------+-----------+----+----------+ +|root.ln.tag1.a.tag2.b.status| null| root.ln| BOOLEAN| PLAIN| SNAPPY|null| null| ++----------------------------+-----+-------------+--------+--------+-----------+----+----------+ +``` + +### insert + +- insert a single column of data + +```sql +IoTDB> insert into root.ln.tag2.d(timestamp,status) values(1,true) +Msg: The statement is executed successfully. +IoTDB> insert into root.ln.tag2.d(timestamp,status) values(2,false) +Msg: The statement is executed successfully. +IoTDB> insert into root.ln.tag2.d(timestamp,status) values(3,true) +Msg: The statement is executed successfully. +IoTDB> insert into root.ln.tag1.a.tag2.d(timestamp,status) values(1,true) +Msg: The statement is executed successfully. +``` + +- insert alignment data + +```sql +IoTDB> insert into root.sg1.tag1.a(time, s1, s2) aligned values(2, 2, 2), (3, 3, 3) +Msg: The statement is executed successfully. +``` + +### select + +- point query + +```sql +IoTDB> select * from root.sg1.tag1.a ++-----------------------------+------------------+------------------+ +| Time|root.sg1.tag1.a.s1|root.sg1.tag1.a.s2| ++-----------------------------+------------------+------------------+ +|1970-01-01T08:00:00.002+08:00| 2.0| 2.0| +|1970-01-01T08:00:00.003+08:00| 3.0| 3.0| ++-----------------------------+------------------+------------------+ +``` + +- align by device + +```sql +IoTDB> select * from root.sg1.tag1.a align by device ++-----------------------------+---------------+---+---+ +| Time| Device| s1| s2| ++-----------------------------+---------------+---+---+ +|1970-01-01T08:00:00.002+08:00|root.sg1.tag1.a|2.0|2.0| +|1970-01-01T08:00:00.003+08:00|root.sg1.tag1.a|3.0|3.0| ++-----------------------------+---------------+---+---+ +``` + +- batch query + +```sql +IoTDB> select status from root.ln.tag2.d.** where time < 2017-11-01T00:08:00.000 ++-----------------------------+----------------------------+---------------------+ +| Time|root.ln.tag1.a.tag2.d.status|root.ln.tag2.d.status| ++-----------------------------+----------------------------+---------------------+ +|1970-01-01T08:00:00.001+08:00| true| true| +|1970-01-01T08:00:00.002+08:00| null| false| +|1970-01-01T08:00:00.003+08:00| null| true| ++-----------------------------+----------------------------+---------------------+ +``` \ No newline at end of file diff --git a/schema-engine-tag/pom.xml b/schema-engine-tag/pom.xml new file mode 100644 index 000000000000..594f70221408 --- /dev/null +++ b/schema-engine-tag/pom.xml @@ -0,0 +1,81 @@ + + + + + iotdb-parent + org.apache.iotdb + 0.14.0-SNAPSHOT + + 4.0.0 + schema-engine-tag + schema-engine-tag + + + org.roaringbitmap + RoaringBitmap + 0.9.32 + + + org.apache.iotdb + iotdb-server + ${project.version} + provided + + + org.reflections + reflections + 0.10.2 + + + + schema-engine-tag + + + org.apache.maven.plugins + maven-assembly-plugin + ${maven.assembly.version} + + + + schema-engine-tag-assembly + package + + single + + + + src/assembly/schema-engine-tag.xml + + false + + + true + true + + + + + + + + + diff --git a/schema-engine-tag/src/assembly/resources/conf/schema-tag.properties b/schema-engine-tag/src/assembly/resources/conf/schema-tag.properties new file mode 100644 index 000000000000..bfb7df48b8ab --- /dev/null +++ b/schema-engine-tag/src/assembly/resources/conf/schema-tag.properties @@ -0,0 +1,33 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +#################### +### tag schema region Configuration +#################### + +# This configuration takes effect only when the schema engine mode is Tag. +# The mode is configured in the 'iotdb-datanode.properties'(schema_engine_mode=Tag). + +# Datatype: int +# The size of wal buffer used to store a wal record.(unit: byte) +# wal_buffer_size = 1024*1024 + +# Datatype: int +# How many device ids a memtable can insert, beyond which the memtable will become immutable +# num_of_deviceIds_in_memTable = 65536 \ No newline at end of file diff --git a/schema-engine-tag/src/assembly/schema-engine-tag.xml b/schema-engine-tag/src/assembly/schema-engine-tag.xml new file mode 100644 index 000000000000..fa8f63ef5da1 --- /dev/null +++ b/schema-engine-tag/src/assembly/schema-engine-tag.xml @@ -0,0 +1,44 @@ + + + + schema-engine-tag + + dir + zip + + false + + + /lib/tag-schema-region + + org.roaringbitmap:RoaringBitmap + org.apache.iotdb:schema-engine-tag + + + + + + src/assembly/resources + ${file.separator} + + + diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java new file mode 100644 index 000000000000..2e5ff69ab607 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegion.java @@ -0,0 +1,986 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +import org.apache.iotdb.common.rpc.thrift.TSchemaNode; +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.file.SystemFileFactory; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException; +import org.apache.iotdb.db.exception.metadata.DataTypeMismatchException; +import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.exception.metadata.SchemaDirCreationFailureException; +import org.apache.iotdb.db.metadata.LocalSchemaProcessor; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; +import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.InsertMeasurementMNode; +import org.apache.iotdb.db.metadata.idtable.entry.SHA256DeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.metadata.mnode.EntityMNode; +import org.apache.iotdb.db.metadata.mnode.IMNode; +import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode; +import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode; +import org.apache.iotdb.db.metadata.plan.schemaregion.write.IActivateTemplateInClusterPlan; +import org.apache.iotdb.db.metadata.plan.schemaregion.write.IActivateTemplatePlan; +import org.apache.iotdb.db.metadata.plan.schemaregion.write.IAutoCreateDeviceMNodePlan; +import org.apache.iotdb.db.metadata.plan.schemaregion.write.ICreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.metadata.plan.schemaregion.write.ICreateTimeSeriesPlan; +import org.apache.iotdb.db.metadata.plan.schemaregion.write.IDeactivateTemplatePlan; +import org.apache.iotdb.db.metadata.plan.schemaregion.write.IPreDeactivateTemplatePlan; +import org.apache.iotdb.db.metadata.plan.schemaregion.write.IRollbackPreDeactivateTemplatePlan; +import org.apache.iotdb.db.metadata.plan.schemaregion.write.ISetTemplatePlan; +import org.apache.iotdb.db.metadata.plan.schemaregion.write.IUnsetTemplatePlan; +import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.metadata.schemaregion.SchemaRegionUtils; +import org.apache.iotdb.db.metadata.tagSchemaRegion.idtable.IDTableWithDeviceIDListImpl; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.TagInvertedIndex; +import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.MeasurementPathUtils; +import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.PathTagConverterUtils; +import org.apache.iotdb.db.metadata.tagSchemaRegion.utils.ShowTimeSeriesResultUtils; +import org.apache.iotdb.db.metadata.template.Template; +import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; +import org.apache.iotdb.db.mpp.common.schematree.MeasurementSchemaInfo; +import org.apache.iotdb.db.qp.physical.crud.InsertPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.ShowDevicesPlan; +import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan; +import org.apache.iotdb.db.query.context.QueryContext; +import org.apache.iotdb.db.query.dataset.ShowDevicesResult; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; +import org.apache.iotdb.external.api.ISeriesNumerMonitor; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.utils.Pair; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +import static org.apache.iotdb.db.utils.EncodingInferenceUtils.getDefaultEncoding; + +/** tag schema region */ +public class TagSchemaRegion implements ISchemaRegion { + private static final Logger logger = LoggerFactory.getLogger(TagSchemaRegion.class); + + protected static IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + + // when a path ends with ".**", it represents batch processing + private final String TAIL = ".**"; + + private final IStorageGroupMNode storageGroupMNode; + private final String storageGroupFullPath; + private final SchemaRegionId schemaRegionId; + private final String schemaRegionDirPath; + + // tag inverted index + private final TagInvertedIndex tagInvertedIndex; + + // manager timeSeries, and use a deviceID list manager device id -> INT32 id + private final IDTableWithDeviceIDListImpl idTableWithDeviceIDList; + + private final ISeriesNumerMonitor seriesNumerMonitor; + + public TagSchemaRegion( + PartialPath storageGroup, + SchemaRegionId schemaRegionId, + IStorageGroupMNode storageGroupMNode, + ISeriesNumerMonitor seriesNumerMonitor) + throws MetadataException { + storageGroupFullPath = storageGroup.getFullPath(); + this.schemaRegionId = schemaRegionId; + String storageGroupDirPath = config.getSchemaDir() + File.separator + storageGroupFullPath; + schemaRegionDirPath = storageGroupDirPath + File.separator + schemaRegionId.getId(); + this.storageGroupMNode = storageGroupMNode; + this.seriesNumerMonitor = seriesNumerMonitor; + File schemaRegionDir = new File(schemaRegionDirPath); + idTableWithDeviceIDList = new IDTableWithDeviceIDListImpl(schemaRegionDir); + tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath); + init(); + } + + @Override + public void init() throws MetadataException { + // must enableIDTableLogFile or deviceIDTransformationMethod=="Plain" + if (!config.isEnableIDTableLogFile() + && config.getDeviceIDTransformationMethod().equals("SHA256")) { + throw new MetadataException( + "enableIDTableLogFile OR deviceIDTransformationMethod==\"Plain\""); + } + File schemaRegionFolder = SystemFileFactory.INSTANCE.getFile(schemaRegionDirPath); + if (!schemaRegionFolder.exists()) { + if (schemaRegionFolder.mkdirs()) { + logger.info("create schema region folder {}", schemaRegionDirPath); + } else { + if (!schemaRegionFolder.exists()) { + logger.error("create schema region folder {} failed.", schemaRegionDirPath); + throw new SchemaDirCreationFailureException(schemaRegionDirPath); + } + } + } + logger.info("initialized successfully: {}", this); + } + + @Override + @TestOnly + public void clear() { + try { + tagInvertedIndex.clear(); + idTableWithDeviceIDList.clear(); + } catch (IOException e) { + logger.error("clear tag inverted index failed", e); + } + } + + @Override + public void forceMlog() { + // no need to record mlog + } + + @Override + public SchemaRegionId getSchemaRegionId() { + return schemaRegionId; + } + + @Override + public String getStorageGroupFullPath() { + return storageGroupFullPath; + } + + @Override + public void deleteSchemaRegion() throws MetadataException { + clear(); + SchemaRegionUtils.deleteSchemaRegionFolder(schemaRegionDirPath, logger); + } + + @Override + public boolean createSnapshot(File snapshotDir) { + // todo implement this + throw new UnsupportedOperationException("Tag mode currently doesn't support snapshot feature."); + } + + @Override + public void loadSnapshot(File latestSnapshotRootDir) { + // todo implement this + throw new UnsupportedOperationException("Tag mode currently doesn't support snapshot feature."); + } + + private void createTagInvertedIndex(PartialPath devicePath) { + Map tagsMap = + PathTagConverterUtils.pathToTags(storageGroupFullPath, devicePath.getFullPath()); + synchronized (idTableWithDeviceIDList) { + tagInvertedIndex.addTags(tagsMap, idTableWithDeviceIDList.size() - 1); + } + } + + private List getDeviceIDsFromInvertedIndex(PartialPath path) { + Map tags = + PathTagConverterUtils.pathToTags(storageGroupFullPath, path.getFullPath()); + return tagInvertedIndex.getMatchedIDs(tags); + } + + private void createTimeseries( + PartialPath path, + TSDataType dataType, + TSEncoding encoding, + CompressionType compressor, + Map props) + throws MetadataException { + createTimeseries( + new CreateTimeSeriesPlan(path, dataType, encoding, compressor, props, null, null, null), 0); + } + + private void createAlignedTimeSeries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries( + new CreateAlignedTimeSeriesPlan( + prefixPath, measurements, dataTypes, encodings, compressors, null, null, null)); + } + + @Override + public void createTimeseries(ICreateTimeSeriesPlan plan, long offset) throws MetadataException { + PartialPath devicePath = plan.getPath().getDevicePath(); + PartialPath path = + new PartialPath( + PathTagConverterUtils.pathToTagsSortPath(storageGroupFullPath, devicePath.getFullPath()) + + "." + + plan.getPath().getMeasurement()); + plan.setPath(path); + devicePath = plan.getPath().getDevicePath(); + DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is not aligned, please use createTimeseries or change entity.", + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } else if (deviceEntry.getMeasurementMap().containsKey(plan.getPath().getMeasurement())) { + throw new PathAlreadyExistException( + devicePath.getFullPath() + "." + plan.getPath().getMeasurement()); + } + } + idTableWithDeviceIDList.createTimeseries(plan); + // write the device path for the first time + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + @Override + public void createAlignedTimeSeries(ICreateAlignedTimeSeriesPlan plan) throws MetadataException { + PartialPath devicePath = plan.getDevicePath(); + PartialPath path = + new PartialPath( + PathTagConverterUtils.pathToTagsSortPath( + storageGroupFullPath, devicePath.getFullPath())); + plan.setDevicePath(path); + devicePath = plan.getDevicePath(); + DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(devicePath.getFullPath()); + if (deviceEntry != null) { + if (!deviceEntry.isAligned()) { + throw new AlignedTimeseriesException( + "Timeseries under this entity is aligned, please use createAlignedTimeseries or change entity.", + devicePath.getFullPath()); + } else { + filterExistingMeasurements(plan, deviceEntry.getMeasurementMap().keySet()); + if (plan.getMeasurements().size() == 0) + throw new PathAlreadyExistException(devicePath.getFullPath()); + } + } + idTableWithDeviceIDList.createAlignedTimeseries(plan); + // write the device path for the first time + if (deviceEntry == null) { + createTagInvertedIndex(devicePath); + } + } + + @Override + public Map checkMeasurementExistence( + PartialPath devicePath, List measurementList, List aliasList) { + throw new UnsupportedOperationException("checkMeasurementExistence"); + } + + private void filterExistingMeasurements( + ICreateAlignedTimeSeriesPlan plan, Set measurementSet) { + List measurements = plan.getMeasurements(); + List dataTypes = plan.getDataTypes(); + List encodings = plan.getEncodings(); + List compressors = plan.getCompressors(); + + List tmpMeasurements = new LinkedList<>(); + List tmpDataTypes = new LinkedList<>(); + List tmpEncodings = new LinkedList<>(); + List tmpCompressors = new LinkedList<>(); + for (int i = 0; i < measurements.size(); i++) { + String measurement = measurements.get(i); + if (!measurementSet.contains(measurement)) { + tmpMeasurements.add(measurements.get(i)); + tmpDataTypes.add(dataTypes.get(i)); + tmpEncodings.add(encodings.get(i)); + tmpCompressors.add(compressors.get(i)); + } + } + plan.setMeasurements(tmpMeasurements); + plan.setDataTypes(tmpDataTypes); + plan.setEncodings(tmpEncodings); + plan.setCompressors(tmpCompressors); + } + + @Override + public Pair> deleteTimeseries(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException("deleteTimeseries"); + } + + @Override + public int constructSchemaBlackList(PathPatternTree patternTree) throws MetadataException { + throw new UnsupportedOperationException("constructSchemaBlackList"); + } + + @Override + public void rollbackSchemaBlackList(PathPatternTree patternTree) throws MetadataException { + throw new UnsupportedOperationException("rollbackSchemaBlackList"); + } + + @Override + public Set fetchSchemaBlackList(PathPatternTree patternTree) + throws MetadataException { + throw new UnsupportedOperationException("fetchSchemaBlackList"); + } + + @Override + public void deleteTimeseriesInBlackList(PathPatternTree patternTree) throws MetadataException { + throw new UnsupportedOperationException("deleteTimeseriesInBlackList"); + } + + @Override + public void autoCreateDeviceMNode(IAutoCreateDeviceMNodePlan plan) throws MetadataException {} + + @Override + public boolean isPathExist(PartialPath path) throws MetadataException { + throw new UnsupportedOperationException("isPathExist"); + } + + @Override + public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + int res = 0; + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + for (IDeviceID deviceID : deviceIDs) { + res += + idTableWithDeviceIDList + .getDeviceEntry(deviceID.toStringID()) + .getMeasurementMap() + .keySet() + .size(); + } + return res; + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, Map templateMap, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException("getAllTimeseriesCount"); + } + + @Override + public int getAllTimeseriesCount( + PartialPath pathPattern, boolean isPrefixMatch, String key, String value, boolean isContains) + throws MetadataException { + throw new UnsupportedOperationException("getAllTimeseriesCount"); + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, int level, boolean isPrefixMatch) throws MetadataException { + throw new UnsupportedOperationException("getMeasurementCountGroupByLevel"); + } + + @Override + public Map getMeasurementCountGroupByLevel( + PartialPath pathPattern, + int level, + boolean isPrefixMatch, + String key, + String value, + boolean isContains) + throws MetadataException { + throw new UnsupportedOperationException("getMeasurementCountGroupByLevel"); + } + + @Override + public int getDevicesNum(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + synchronized (idTableWithDeviceIDList) { + if (pathPattern.getFullPath().length() <= storageGroupFullPath.length()) { + return idTableWithDeviceIDList.size(); + } else { + return getDeviceIDsFromInvertedIndex(pathPattern).size(); + } + } + } + + @Override + public int getNodesCountInGivenLevel(PartialPath pathPattern, int level, boolean isPrefixMatch) + throws MetadataException { + throw new UnsupportedOperationException("getNodesCountInGivenLevel"); + } + + @Override + public List getNodesListInGivenLevel( + PartialPath pathPattern, + int nodeLevel, + boolean isPrefixMatch, + LocalSchemaProcessor.StorageGroupFilter filter) + throws MetadataException { + throw new UnsupportedOperationException("getNodesListInGivenLevel"); + } + + @Override + public Set getChildNodePathInNextLevel(PartialPath pathPattern) + throws MetadataException { + throw new UnsupportedOperationException("getChildNodePathInNextLevel"); + } + + @Override + public Set getChildNodeNameInNextLevel(PartialPath pathPattern) throws MetadataException { + throw new UnsupportedOperationException("getChildNodeNameInNextLevel"); + } + + @Override + public Set getBelongedDevices(PartialPath timeseries) throws MetadataException { + throw new UnsupportedOperationException("getBelongedDevices"); + } + + @Override + public Set getMatchedDevices(PartialPath pathPattern, boolean isPrefixMatch) + throws MetadataException { + List deviceIDs = getDeviceIdFromInvertedIndex(pathPattern); + Set matchedDevices = new HashSet<>(); + String devicePath = pathPattern.getFullPath(); + // exact query + if (!devicePath.endsWith(TAIL) && !devicePath.equals(storageGroupFullPath)) { + DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(devicePath); + if (deviceEntry != null) { + matchedDevices.add(pathPattern); + } + return matchedDevices; + } + List devicePaths = getDevicePaths(deviceIDs); + for (String path : devicePaths) { + matchedDevices.add(new PartialPath(path)); + } + return matchedDevices; + } + + private List getDevicePaths(List deviceIDS) { + List devicePaths = new ArrayList<>(); + if (config.getDeviceIDTransformationMethod().equals("SHA256")) { + List schemaEntries = new ArrayList<>(); + for (IDeviceID deviceID : deviceIDS) { + DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(deviceID.toStringID()); + Map map = deviceEntry.getMeasurementMap(); + // For each device, only one SchemaEntry needs to be obtained + for (Map.Entry entry : map.entrySet()) { + schemaEntries.add(entry.getValue()); + break; + } + } + List diskSchemaEntries = + idTableWithDeviceIDList.getDiskSchemaEntries(schemaEntries); + for (DiskSchemaEntry diskSchemaEntry : diskSchemaEntries) { + devicePaths.add(diskSchemaEntry.getDevicePath()); + } + } else { + for (IDeviceID deviceID : deviceIDS) { + devicePaths.add(deviceID.toStringID()); + } + } + return devicePaths; + } + + private List getSchemaEntries(List deviceIDS) { + List schemaEntries = new ArrayList<>(); + for (IDeviceID deviceID : deviceIDS) { + DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(deviceID.toStringID()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (Map.Entry entry : schemaMap.entrySet()) { + schemaEntries.add(entry.getValue()); + } + } + return schemaEntries; + } + + private List getMeasurementPaths(List deviceIDS) + throws IllegalPathException { + List measurementPaths = new ArrayList<>(); + if (config.getDeviceIDTransformationMethod().equals("SHA256")) { + List schemaEntries = getSchemaEntries(deviceIDS); + List diskSchemaEntries = + idTableWithDeviceIDList.getDiskSchemaEntries(schemaEntries); + for (DiskSchemaEntry diskSchemaEntry : diskSchemaEntries) { + MeasurementPath measurementPath = + MeasurementPathUtils.generateMeasurementPath(diskSchemaEntry); + measurementPaths.add(measurementPath); + } + } else { + for (IDeviceID deviceID : deviceIDS) { + DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(deviceID.toStringID()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (Map.Entry entry : schemaMap.entrySet()) { + MeasurementPath measurementPath = + MeasurementPathUtils.generateMeasurementPath( + deviceID.toStringID(), entry.getKey(), entry.getValue(), deviceEntry.isAligned()); + measurementPaths.add(measurementPath); + } + } + } + return measurementPaths; + } + + @Override + public Pair, Integer> getMatchedDevices(ShowDevicesPlan plan) + throws MetadataException { + throw new UnsupportedOperationException("getMatchedDevices"); + } + + @Override + public List getMeasurementPaths( + PartialPath pathPattern, boolean isPrefixMatch, boolean withTags) throws MetadataException { + PartialPath devicePath = pathPattern.getDevicePath(); + if (devicePath.getFullPath().endsWith(TAIL)) { + return getMeasurementPathsWithBatchQuery(devicePath, isPrefixMatch); + } else { + return getMeasurementPathsWithPointQuery(devicePath, isPrefixMatch); + } + } + + @Override + public Pair, Integer> getMeasurementPathsWithAlias( + PartialPath pathPattern, int limit, int offset, boolean isPrefixMatch, boolean withTags) + throws MetadataException { + List res = getMeasurementPaths(pathPattern, isPrefixMatch, false); + Pair, Integer> result = new Pair<>(res, 0); + return result; + } + + @Override + public List fetchSchema( + PartialPath pathPattern, Map templateMap, boolean withTags) + throws MetadataException { + return null; + } + + private List getMeasurementPathsWithPointQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List measurementPaths = new LinkedList<>(); + String path = + PathTagConverterUtils.pathToTagsSortPath(storageGroupFullPath, devicePath.getFullPath()); + DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(path); + if (deviceEntry == null) return measurementPaths; + Map schemaMap = deviceEntry.getMeasurementMap(); + for (Map.Entry entry : schemaMap.entrySet()) { + MeasurementPath measurementPath = + MeasurementPathUtils.generateMeasurementPath( + path, entry.getKey(), entry.getValue(), deviceEntry.isAligned()); + measurementPaths.add(measurementPath); + } + return measurementPaths; + } + + private List getMeasurementPathsWithBatchQuery( + PartialPath devicePath, boolean isPrefixMatch) throws MetadataException { + List deviceIDs = getDeviceIdFromInvertedIndex(devicePath); + return getMeasurementPaths(deviceIDs); + } + + @Override + public Pair, Integer> showTimeseries( + ShowTimeSeriesPlan plan, QueryContext context) throws MetadataException { + List ShowTimeSeriesResults = new ArrayList<>(); + Pair, Integer> result = new Pair<>(ShowTimeSeriesResults, 0); + String path = plan.getPath().getFullPath(); + // point query + if (!path.endsWith(TAIL)) { + path = PathTagConverterUtils.pathToTagsSortPath(storageGroupFullPath, path); + DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(path); + if (deviceEntry != null) { + Map measurementMap = deviceEntry.getMeasurementMap(); + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + ShowTimeSeriesResults.add( + ShowTimeSeriesResultUtils.generateShowTimeSeriesResult( + storageGroupFullPath, path, m, schemaEntry)); + } + } + return result; + } + // batch query + List deviceIDs = getDeviceIdFromInvertedIndex(plan.getPath()); + for (IDeviceID deviceID : deviceIDs) { + getTimeSeriesResultOfDeviceFromIDTable(ShowTimeSeriesResults, deviceID); + } + return result; + } + + private void getTimeSeriesResultOfDeviceFromIDTable( + List ShowTimeSeriesResults, IDeviceID deviceID) { + Map measurementMap = + idTableWithDeviceIDList.getDeviceEntry(deviceID.toStringID()).getMeasurementMap(); + if (deviceID instanceof SHA256DeviceID) { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + List schemaEntries = new ArrayList<>(); + schemaEntries.add(schemaEntry); + List diskSchemaEntries = + idTableWithDeviceIDList.getDiskSchemaEntries(schemaEntries); + DiskSchemaEntry diskSchemaEntry = diskSchemaEntries.get(0); + ShowTimeSeriesResults.add( + ShowTimeSeriesResultUtils.generateShowTimeSeriesResult( + storageGroupFullPath, diskSchemaEntry.seriesKey, schemaEntry)); + } + } else { + for (String m : measurementMap.keySet()) { + SchemaEntry schemaEntry = measurementMap.get(m); + ShowTimeSeriesResults.add( + ShowTimeSeriesResultUtils.generateShowTimeSeriesResult( + storageGroupFullPath, deviceID.toStringID(), m, schemaEntry)); + } + } + } + + private List getDeviceIdFromInvertedIndex(PartialPath devicePath) + throws MetadataException { + String path = devicePath.getFullPath(); + if (path.endsWith(TAIL)) { + path = path.substring(0, path.length() - TAIL.length()); + devicePath = new PartialPath(path); + } + synchronized (idTableWithDeviceIDList) { + if (devicePath.getFullPath().length() <= storageGroupFullPath.length()) { + return idTableWithDeviceIDList.getAllDeviceIDS(); + } else { + List IDS = new LinkedList<>(); + List ids = getDeviceIDsFromInvertedIndex(devicePath); + if (ids.size() > 0) { + for (int id : ids) { + IDS.add(idTableWithDeviceIDList.get(id)); + } + } + return IDS; + } + } + } + + @Override + public List getAllMeasurementByDevicePath(PartialPath devicePath) + throws PathNotExistException { + throw new UnsupportedOperationException("getAllMeasurementByDevicePath"); + } + + @Override + public IMNode getDeviceNode(PartialPath path) throws MetadataException { + DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(path.getFullPath()); + if (deviceEntry == null) throw new PathNotExistException(path.getFullPath()); + return new EntityMNode(storageGroupMNode, path.getFullPath()); + } + + @Override + public IMeasurementMNode getMeasurementMNode(PartialPath fullPath) throws MetadataException { + throw new UnsupportedOperationException("getMeasurementMNode"); + } + + @Override + public void changeAlias(PartialPath path, String alias) throws MetadataException, IOException { + throw new UnsupportedOperationException("changeAlias"); + } + + @Override + public void upsertTagsAndAttributes( + String alias, + Map tagsMap, + Map attributesMap, + PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException("upsertTagsAndAttributes"); + } + + @Override + public void addAttributes(Map attributesMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException("addAttributes"); + } + + @Override + public void addTags(Map tagsMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException("addTags"); + } + + @Override + public void dropTagsOrAttributes(Set keySet, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException("dropTagsOrAttributes"); + } + + @Override + public void setTagsOrAttributesValue(Map alterMap, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException("setTagsOrAttributesValue"); + } + + @Override + public void renameTagOrAttributeKey(String oldKey, String newKey, PartialPath fullPath) + throws MetadataException, IOException { + throw new UnsupportedOperationException("renameTagOrAttributeKey"); + } + + @Override + public IMNode getSeriesSchemasAndReadLockDevice(InsertPlan plan) + throws MetadataException, IOException { + PartialPath devicePath = plan.getDevicePath(); + devicePath = + new PartialPath( + PathTagConverterUtils.pathToTagsSortPath( + storageGroupFullPath, devicePath.getFullPath())); + plan.setDevicePath(devicePath); + String[] measurementList = plan.getMeasurements(); + IMeasurementMNode[] measurementMNodes = plan.getMeasurementMNodes(); + checkAlignedAndAutoCreateSeries(plan); + IMNode deviceMNode = getDeviceNode(devicePath); + IMeasurementMNode measurementMNode; + DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(devicePath.getFullPath()); + Map schemaMap = deviceEntry.getMeasurementMap(); + for (int i = 0; i < measurementList.length; i++) { + SchemaEntry schemaEntry = schemaMap.get(measurementList[i]); + measurementMNode = new InsertMeasurementMNode(measurementList[i], schemaEntry, null); + // check type is match + try { + SchemaRegionUtils.checkDataTypeMatch(plan, i, schemaEntry.getTSDataType()); + } catch (DataTypeMismatchException mismatchException) { + if (!config.isEnablePartialInsert()) { + throw mismatchException; + } else { + // mark failed measurement + plan.markFailedMeasurementInsertion(i, mismatchException); + continue; + } + } + measurementMNodes[i] = measurementMNode; + } + plan.setDeviceID(deviceEntry.getDeviceID()); + plan.setDevicePath(new PartialPath(deviceEntry.getDeviceID().toStringID(), false)); + return deviceMNode; + } + + @Override + public DeviceSchemaInfo getDeviceSchemaInfoWithAutoCreate( + PartialPath devicePath, + String[] measurements, + Function getDataType, + TSEncoding[] encodings, + CompressionType[] compressionTypes, + boolean aligned) + throws MetadataException { + List measurementSchemaInfoList = new ArrayList<>(measurements.length); + for (int i = 0; i < measurements.length; i++) { + SchemaEntry schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + if (schemaEntry == null) { + if (config.isAutoCreateSchemaEnabled()) { + if (aligned) { + TSDataType dataType = getDataType.apply(i); + internalAlignedCreateTimeseries( + devicePath, + Collections.singletonList(measurements[i]), + Collections.singletonList(dataType), + Collections.singletonList( + encodings[i] == null ? getDefaultEncoding(dataType) : encodings[i]), + Collections.singletonList( + compressionTypes[i] == null + ? TSFileDescriptor.getInstance().getConfig().getCompressor() + : compressionTypes[i])); + + } else { + internalCreateTimeseries( + devicePath.concatNode(measurements[i]), + getDataType.apply(i), + encodings[i], + compressionTypes[i]); + } + } + schemaEntry = getSchemaEntry(devicePath.getFullPath(), measurements[i]); + } + measurementSchemaInfoList.add( + new MeasurementSchemaInfo( + measurements[i], + new MeasurementSchema( + measurements[i], + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType()), + null)); + } + return new DeviceSchemaInfo(devicePath, aligned, measurementSchemaInfoList); + } + + private SchemaEntry getSchemaEntry(String devicePath, String measurementName) { + DeviceEntry deviceEntry = idTableWithDeviceIDList.getDeviceEntry(devicePath); + if (deviceEntry == null) return null; + return deviceEntry.getSchemaEntry(measurementName); + } + + private void checkAlignedAndAutoCreateSeries(InsertPlan plan) throws MetadataException { + String[] measurementList = plan.getMeasurements(); + try { + if (plan.isAligned()) { + internalAlignedCreateTimeseries( + plan.getDevicePath(), + Arrays.asList(measurementList), + Arrays.asList(plan.getDataTypes())); + } else { + internalCreateTimeseries( + plan.getDevicePath().concatNode(measurementList[0]), plan.getDataTypes()[0]); + } + } catch (MetadataException e) { + if (!(e instanceof PathAlreadyExistException)) { + throw e; + } + } + } + + /** create timeseries ignoring PathAlreadyExistException */ + private void internalCreateTimeseries(PartialPath path, TSDataType dataType) + throws MetadataException { + createTimeseries( + path, + dataType, + getDefaultEncoding(dataType), + TSFileDescriptor.getInstance().getConfig().getCompressor(), + Collections.emptyMap()); + } + + /** create timeseries ignoring PathAlreadyExistException */ + private void internalCreateTimeseries( + PartialPath path, TSDataType dataType, TSEncoding encoding, CompressionType compressor) + throws MetadataException { + if (encoding == null) { + encoding = getDefaultEncoding(dataType); + } + if (compressor == null) { + compressor = TSFileDescriptor.getInstance().getConfig().getCompressor(); + } + createTimeseries(path, dataType, encoding, compressor, Collections.emptyMap()); + } + + /** create aligned timeseries ignoring PathAlreadyExistException */ + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, List measurements, List dataTypes) + throws MetadataException { + List encodings = new ArrayList<>(); + List compressors = new ArrayList<>(); + for (TSDataType dataType : dataTypes) { + encodings.add(getDefaultEncoding(dataType)); + compressors.add(TSFileDescriptor.getInstance().getConfig().getCompressor()); + } + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); + } + + /** create aligned timeseries ignoring PathAlreadyExistException */ + private void internalAlignedCreateTimeseries( + PartialPath prefixPath, + List measurements, + List dataTypes, + List encodings, + List compressors) + throws MetadataException { + createAlignedTimeSeries(prefixPath, measurements, dataTypes, encodings, compressors); + } + + @Override + public Set getPathsSetTemplate(String templateName) throws MetadataException { + throw new UnsupportedOperationException("getPathsSetTemplate"); + } + + @Override + public Set getPathsUsingTemplate(String templateName) throws MetadataException { + throw new UnsupportedOperationException("getPathsUsingTemplate"); + } + + @Override + public boolean isTemplateAppendable(Template template, List measurements) + throws MetadataException { + throw new UnsupportedOperationException("isTemplateAppendable"); + } + + @Override + public void setSchemaTemplate(ISetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException("setSchemaTemplate"); + } + + @Override + public void unsetSchemaTemplate(IUnsetTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException("unsetSchemaTemplate"); + } + + @Override + public void setUsingSchemaTemplate(IActivateTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException("setUsingSchemaTemplate"); + } + + @Override + public void activateSchemaTemplate(IActivateTemplateInClusterPlan plan, Template template) + throws MetadataException { + throw new UnsupportedOperationException("activateSchemaTemplate"); + } + + @Override + public List getPathsUsingTemplate(PartialPath pathPattern, int templateId) + throws MetadataException { + throw new UnsupportedOperationException("getPathsUsingTemplate"); + } + + @Override + public int constructSchemaBlackListWithTemplate(IPreDeactivateTemplatePlan plan) + throws MetadataException { + throw new UnsupportedOperationException("constructSchemaBlackListWithTemplate"); + } + + @Override + public void rollbackSchemaBlackListWithTemplate(IRollbackPreDeactivateTemplatePlan plan) + throws MetadataException { + throw new UnsupportedOperationException("rollbackSchemaBlackListWithTemplate"); + } + + @Override + public void deactivateTemplateInBlackList(IDeactivateTemplatePlan plan) throws MetadataException { + throw new UnsupportedOperationException("deactivateTemplateInBlackList"); + } + + @Override + public IMNode getMNodeForTrigger(PartialPath fullPath) throws MetadataException { + throw new UnsupportedOperationException("getMNodeForTrigger"); + } + + @Override + public void releaseMNodeAfterDropTrigger(IMNode node) throws MetadataException { + throw new UnsupportedOperationException("releaseMNodeAfterDropTrigger"); + } + + @Override + public String toString() { + return "TagSchemaRegion{" + + "storageGroupFullPath='" + + storageGroupFullPath + + '\'' + + ", schemaRegionId=" + + schemaRegionId + + ", schemaRegionDirPath='" + + schemaRegionDirPath + + '\'' + + '}'; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/config/TagSchemaConfig.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/config/TagSchemaConfig.java new file mode 100644 index 000000000000..74e5bb41fd98 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/config/TagSchemaConfig.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.config; + +/** tag schema region config */ +public class TagSchemaConfig { + + // the maximum number of device ids managed by a working memTable + private int numOfDeviceIdsInMemTable = 65536; + + // the size of wal buffer used to store a wal record + private int walBufferSize = 1024 * 1024; + + public int getNumOfDeviceIdsInMemTable() { + return numOfDeviceIdsInMemTable; + } + + public void setNumOfDeviceIdsInMemTable(int numOfDeviceIdsInMemTable) { + this.numOfDeviceIdsInMemTable = numOfDeviceIdsInMemTable; + } + + public int getWalBufferSize() { + return walBufferSize; + } + + public void setWalBufferSize(int walBufferSize) { + this.walBufferSize = walBufferSize; + } + + @Override + public String toString() { + return "TagSchemaConfig[" + + "numOfDeviceIdsInMemTable=" + + numOfDeviceIdsInMemTable + + ", walBufferSize=" + + walBufferSize + + "]"; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/config/TagSchemaDescriptor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/config/TagSchemaDescriptor.java new file mode 100644 index 000000000000..f6b96de87d00 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/config/TagSchemaDescriptor.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.config; + +import org.apache.iotdb.commons.conf.IoTDBConstant; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +/** manager tag schema config */ +public class TagSchemaDescriptor { + private static final Logger logger = LoggerFactory.getLogger(TagSchemaDescriptor.class); + + private static final String TAG_SCHEMA_CONFIG_FILE_NAME = "schema-tag.properties"; + + private final TagSchemaConfig conf = new TagSchemaConfig(); + + private TagSchemaDescriptor() { + loadProperties(); + } + + public static TagSchemaDescriptor getInstance() { + return TagSchemaDescriptorHolder.INSTANCE; + } + + private void loadProperties() { + String iotDBHomePath = System.getProperty(IoTDBConstant.IOTDB_HOME, null); + if (iotDBHomePath == null) { + logger.warn( + "Cannot find IOTDB_HOME environment variable when loading " + + "config file {}, use default configuration", + TAG_SCHEMA_CONFIG_FILE_NAME); + return; + } + String tagSchemaConfigPath = + iotDBHomePath + + File.separatorChar + + "conf" + + File.separatorChar + + TAG_SCHEMA_CONFIG_FILE_NAME; + try (InputStream in = new BufferedInputStream(new FileInputStream(tagSchemaConfigPath))) { + Properties properties = new Properties(); + properties.load(in); + conf.setWalBufferSize( + Integer.parseInt( + properties.getProperty("wal_buffer_size", String.valueOf(conf.getWalBufferSize())))); + conf.setNumOfDeviceIdsInMemTable( + Integer.parseInt( + properties.getProperty( + "num_of_deviceIds_in_memTable", + String.valueOf(conf.getNumOfDeviceIdsInMemTable())))); + } catch (FileNotFoundException e) { + logger.warn("Fail to find tag schema region config file {}", tagSchemaConfigPath); + } catch (IOException e) { + logger.warn("Cannot load tag schema region config file, use default configuration"); + } + } + + public TagSchemaConfig getTagSchemaConfig() { + return conf; + } + + private static class TagSchemaDescriptorHolder { + + private static final TagSchemaDescriptor INSTANCE = new TagSchemaDescriptor(); + + private TagSchemaDescriptorHolder() {} + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImpl.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImpl.java new file mode 100644 index 000000000000..31b0df4888f3 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImpl.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.idtable; + +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.metadata.idtable.IDTableHashmapImpl; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry; +import org.apache.iotdb.db.metadata.idtable.entry.DeviceIDFactory; +import org.apache.iotdb.db.metadata.idtable.entry.IDeviceID; +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.metadata.plan.schemaregion.write.ICreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.metadata.plan.schemaregion.write.ICreateTimeSeriesPlan; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * an idTable implementation，manager timeSeries, and use a deviceID list manager device id -> INT32 + * id + */ +public class IDTableWithDeviceIDListImpl extends IDTableHashmapImpl { + + // use a deviceID list manager device id -> INT32 id + private List deviceIDS; + + public IDTableWithDeviceIDListImpl(File storageGroupDir) { + super(storageGroupDir); + if (deviceIDS == null) { + deviceIDS = new ArrayList<>(); + } + } + + /** + * Whether device entry already exists + * + * @param devicePath device path + * @return false if the device entry already exists, otherwise return true + */ + private boolean deviceEntryNotExist(String devicePath) { + DeviceEntry deviceEntry = getDeviceEntry(devicePath); + return deviceEntry == null; + } + /** + * create aligned timeseries + * + * @param plan create aligned timeseries plan + * @throws MetadataException + */ + @Override + public synchronized void createAlignedTimeseries(ICreateAlignedTimeSeriesPlan plan) + throws MetadataException { + String devicePath = plan.getDevicePath().getFullPath(); + // This device path is created for the first time, append it to deviceIdList + if (deviceEntryNotExist(devicePath)) { + IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath); + deviceIDS.add(deviceID); + } + super.createAlignedTimeseries(plan); + } + + /** + * create timeseries + * + * @param plan create timeseries plan + * @throws MetadataException + */ + @Override + public synchronized void createTimeseries(ICreateTimeSeriesPlan plan) throws MetadataException { + String devicePath = plan.getPath().getDevicePath().getFullPath(); + // This device path is created for the first time, append it to deviceIdList + if (deviceEntryNotExist(devicePath)) { + IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath); + deviceIDS.add(deviceID); + } + super.createTimeseries(plan); + } + + /** + * put schema entry to id table, currently used in recover + * + * @param devicePath device path (can be device id formed path) + * @param measurement measurement name + * @param schemaEntry schema entry to put + * @param isAligned is the device aligned + */ + @Override + public void putSchemaEntry( + String devicePath, String measurement, SchemaEntry schemaEntry, boolean isAligned) + throws MetadataException { + if (deviceIDS == null) { + deviceIDS = new ArrayList<>(); + } + // This device path is created for the first time, append it to deviceIdList + if (deviceEntryNotExist(devicePath)) { + IDeviceID deviceID = DeviceIDFactory.getInstance().getDeviceID(devicePath); + deviceIDS.add(deviceID); + } + super.putSchemaEntry(devicePath, measurement, schemaEntry, isAligned); + } + + @Override + @TestOnly + public void clear() throws IOException { + super.clear(); + deviceIDS = null; + } + + public synchronized IDeviceID get(int index) { + return deviceIDS.get(index); + } + + public int size() { + return deviceIDS.size(); + } + + public List getAllDeviceIDS() { + return new ArrayList<>(deviceIDS); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java new file mode 100644 index 000000000000..9dab290d900b --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/ITagInvertedIndex.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +import org.apache.iotdb.commons.utils.TestOnly; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** tag inverted index interface */ +public interface ITagInvertedIndex { + + /** + * insert tags and device id + * + * @param tags tags like: + * @param id INT32 device id + */ + void addTags(Map tags, int id); + + /** + * delete tags and id using delete request context + * + * @param tags tags like: + * @param id INT32 device id + */ + void removeTags(Map tags, int id); + + /** + * get all matching device ids + * + * @param tags tags like: + * @return device ids + */ + List getMatchedIDs(Map tags); + + /** + * Close all open resources + * + * @throws IOException + */ + @TestOnly + void clear() throws IOException; +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java new file mode 100644 index 000000000000..1feab4c92cf1 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/DeletionRequest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request; + +import org.apache.iotdb.lsm.context.requestcontext.RequestContext; +import org.apache.iotdb.lsm.request.IDeletionRequest; + +import java.util.List; + +/** Represents a deletion request */ +public class DeletionRequest implements IDeletionRequest { + + // tags + List keys; + + // int32 id + int value; + + public DeletionRequest(List keys, int value) { + super(); + this.keys = keys; + this.value = value; + } + + @Override + public String getKey(RequestContext context) { + return keys.get(context.getLevel() - 1); + } + + @Override + public List getKeys() { + return keys; + } + + @Override + public Integer getValue() { + return value; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java new file mode 100644 index 000000000000..e8c80cd8927d --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/InsertionRequest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request; + +import org.apache.iotdb.lsm.context.requestcontext.RequestContext; +import org.apache.iotdb.lsm.request.IInsertionRequest; + +import java.util.List; + +/** Represents a insertion request */ +public class InsertionRequest implements IInsertionRequest { + + // tags + List keys; + + // int32 id + int value; + + public InsertionRequest() { + super(); + } + + public InsertionRequest(List keys, int value) { + super(); + this.keys = keys; + this.value = value; + } + + @Override + public String getKey(RequestContext context) { + return keys.get(context.getLevel() - 1); + } + + @Override + public List getKeys() { + return keys; + } + + @Override + public Integer getValue() { + return value; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java new file mode 100644 index 000000000000..6d4bfe462304 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/Request/QueryRequest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request; + +import org.apache.iotdb.lsm.context.requestcontext.RequestContext; +import org.apache.iotdb.lsm.request.IQueryRequest; + +import java.util.List; + +/** Represents a query request */ +public class QueryRequest implements IQueryRequest { + + // tags + List keys; + + public QueryRequest(List keys) { + super(); + this.keys = keys; + } + + @Override + public String getKey(RequestContext context) { + return keys.get(context.getLevel() - 1); + } + + @Override + public List getKeys() { + return keys; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java new file mode 100644 index 000000000000..a0bc36922282 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagInvertedIndex.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaConfig; +import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaDescriptor; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.response.QueryResponse; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALEntry; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal.WALManager; +import org.apache.iotdb.lsm.engine.LSMEngine; +import org.apache.iotdb.lsm.engine.LSMEngineBuilder; + +import org.roaringbitmap.RoaringBitmap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** tag inverted index, tag is and id is int32 auto increment id */ +public class TagInvertedIndex implements ITagInvertedIndex { + + // This file records the wal log + private static final String WAL_FILE_NAME = "tag_inverted_index.log"; + + private static final Logger logger = LoggerFactory.getLogger(TagInvertedIndex.class); + + // Manage configuration information of tag schema region + private static final TagSchemaConfig tagSchemaConfig = + TagSchemaDescriptor.getInstance().getTagSchemaConfig(); + + // Directly use the lsm engine that comes with the lsm framework to implement the tag inverted + // index + LSMEngine lsmEngine; + + /** + * initialization method + * + * @param schemaDirPath schema dirPath + */ + public TagInvertedIndex(String schemaDirPath) { + try { + WALManager walManager = + new WALManager( + schemaDirPath, + WAL_FILE_NAME, + tagSchemaConfig.getWalBufferSize(), + new WALEntry(), + false); + // root memory node, used to manage working and immutableMemTables + MemTableGroup memTableGroup = + new MemTableGroup(tagSchemaConfig.getNumOfDeviceIdsInMemTable()); + + // build lsm engine + lsmEngine = + new LSMEngineBuilder() + .buildLSMManagers("org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex", walManager) + .buildRootMemNode(memTableGroup) + .build(); + + // recover the lsm engine + lsmEngine.recover(); + } catch (IOException e) { + logger.info("TagInvertedIndex initialization failed"); + logger.error(e.getMessage()); + } + } + + /** + * insert tags and device id + * + * @param tags tags like: + * @param id INT32 device id + */ + @Override + public synchronized void addTags(Map tags, int id) { + for (Map.Entry tag : tags.entrySet()) { + InsertionRequest insertionRequest = + new InsertionRequest(generateKeys(tag.getKey(), tag.getValue()), id); + lsmEngine.insert(insertionRequest); + } + } + + /** + * delete tags and id using delete request context + * + * @param tags tags like: + * @param id INT32 device id + */ + @Override + public synchronized void removeTags(Map tags, int id) { + for (Map.Entry tag : tags.entrySet()) { + DeletionRequest deletionRequest = + new DeletionRequest(generateKeys(tag.getKey(), tag.getValue()), id); + lsmEngine.delete(deletionRequest); + } + } + + /** + * get all matching device ids + * + * @param tags tags like: + * @return ids + */ + @Override + public synchronized List getMatchedIDs(Map tags) { + RoaringBitmap roaringBitmap = new RoaringBitmap(); + int i = 0; + for (Map.Entry tag : tags.entrySet()) { + RoaringBitmap rb = getMatchedIDs(tag.getKey(), tag.getValue()); + if (rb == null) continue; + else { + if (i == 0) roaringBitmap = rb; + else roaringBitmap = RoaringBitmap.and(roaringBitmap, rb); + i++; + } + } + return Arrays.stream(roaringBitmap.toArray()).boxed().collect(Collectors.toList()); + } + + /** + * Generate the keys in the request + * + * @param tagKey tag key + * @param tagValue tag value + * @return keys + */ + private List generateKeys(String tagKey, String tagValue) { + List keys = new ArrayList<>(); + keys.add(tagKey); + keys.add(tagValue); + return keys; + } + + /** + * Get ids matching the tag + * + * @param tagKey tag key + * @param tagValue tag value + * @return roaring bitmap + */ + private RoaringBitmap getMatchedIDs(String tagKey, String tagValue) { + QueryRequest queryRequest = new QueryRequest(generateKeys(tagKey, tagValue)); + QueryResponse response = lsmEngine.query(queryRequest); + return response.getValue(); + } + + /** + * Close all open resources + * + * @throws IOException + */ + @Override + @TestOnly + public void clear() throws IOException { + lsmEngine.clear(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java new file mode 100644 index 000000000000..7e0d70d0239a --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkDeletion.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.lsm.annotation.DeletionProcessor; +import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcessor; + +import java.util.List; + +/** deletion for MemChunk */ +@DeletionProcessor(level = 3) +public class MemChunkDeletion extends DeleteLevelProcessor { + + /** + * MemChunk is the last layer of memory nodes, no children + * + * @param memNode memory node + * @param context request context + * @return null + */ + @Override + public List getChildren( + MemChunk memNode, DeletionRequest request, DeleteRequestContext context) { + return null; + } + + /** + * the delete method corresponding to the MemChunk node + * + * @param memNode memory node + * @param context deletion request context + */ + @Override + public void delete(MemChunk memNode, DeletionRequest request, DeleteRequestContext context) { + Integer deviceID = request.getValue(); + memNode.remove(deviceID); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java new file mode 100644 index 000000000000..c6718d941739 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemChunkGroupDeletion.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.lsm.annotation.DeletionProcessor; +import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcessor; + +import java.util.ArrayList; +import java.util.List; + +/** deletion for MemChunkGroup */ +@DeletionProcessor(level = 2) +public class MemChunkGroupDeletion + extends DeleteLevelProcessor { + + /** + * get all MemChunks that need to be processed in the current MemChunkGroup + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunks + */ + @Override + public List getChildren( + MemChunkGroup memNode, DeletionRequest deletionRequest, DeleteRequestContext context) { + List memChunks = new ArrayList<>(); + String tagValue = deletionRequest.getKey(context); + MemChunk child = memNode.get(tagValue); + if (child != null) memChunks.add(child); + return memChunks; + } + + /** + * the delete method corresponding to the MemChunkGroup node + * + * @param memNode memory node + * @param context deletion request context + */ + @Override + public void delete( + MemChunkGroup memNode, DeletionRequest deletionRequest, DeleteRequestContext context) { + String tagValue = deletionRequest.getKey(context); + MemChunk child = memNode.get(tagValue); + if (child == null || child.isEmpty()) { + memNode.remove(tagValue); + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java new file mode 100644 index 000000000000..fa27bba523f8 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableDeletion.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.annotation.DeletionProcessor; +import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcessor; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** deletion for MemTable */ +@DeletionProcessor(level = 1) +public class MemTableDeletion + extends DeleteLevelProcessor { + + /** + * get all MemChunkGroups that need to be processed in the current MemTable + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunkGroups + */ + @Override + public List getChildren( + MemTable memNode, DeletionRequest deletionRequest, DeleteRequestContext context) { + if (memNode.isImmutable()) return new ArrayList<>(); + List memChunkGroups = new ArrayList<>(); + String tagKey = deletionRequest.getKey(context); + MemChunkGroup child = memNode.get(tagKey); + if (child != null) memChunkGroups.add(child); + return memChunkGroups; + } + + /** + * the delete method corresponding to the MemTable node + * + * @param memNode memory node + * @param context deletion request context + */ + @Override + public void delete( + MemTable memNode, DeletionRequest deletionRequest, DeleteRequestContext context) { + if (memNode.isImmutable()) { + Set deletionList = memNode.getDeletionList(); + if (!deletionList.contains(deletionRequest.getValue())) { + deletionList.add(deletionRequest.getValue()); + } + return; + } + String tagKey = deletionRequest.getKey(context); + MemChunkGroup child = memNode.get(tagKey); + if (child == null || child.isEmpty()) { + memNode.remove(tagKey); + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java new file mode 100644 index 000000000000..249dd9acfb5b --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/deletion/MemTableGroupDeletion.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.deletion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup; +import org.apache.iotdb.lsm.annotation.DeletionProcessor; +import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext; +import org.apache.iotdb.lsm.levelProcess.DeleteLevelProcessor; + +import java.util.ArrayList; +import java.util.List; + +/** deletion for MemTableGroup */ +@DeletionProcessor(level = 0) +public class MemTableGroupDeletion + extends DeleteLevelProcessor { + + /** + * get all MemTables that need to be processed in the current MemTableGroup + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemTables + */ + @Override + public List getChildren( + MemTableGroup memNode, DeletionRequest request, DeleteRequestContext context) { + List memTables = new ArrayList<>(); + int id = request.getValue(); + if (memNode.inWorkingMemTable(id)) { + memTables.add(memNode.getWorkingMemTable()); + } else { + memTables.add( + memNode.getImmutableMemTables().get(id / memNode.getNumOfDeviceIdsInMemTable())); + } + return memTables; + } + + /** + * the delete method corresponding to the MemTableGroup node, do nothing + * + * @param memNode memory node + * @param context deletion request context + */ + @Override + public void delete( + MemTableGroup memNode, DeletionRequest request, DeleteRequestContext context) {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java new file mode 100644 index 000000000000..9abf201dc967 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkGroupInsertion.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.lsm.annotation.InsertionProcessor; +import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcessor; + +import java.util.ArrayList; +import java.util.List; + +/** insertion for MemChunkGroup */ +@InsertionProcessor(level = 2) +public class MemChunkGroupInsertion + extends InsertLevelProcessor { + + /** + * get all MemChunks that need to be processed in the current MemChunkGroup + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunks + */ + @Override + public List getChildren( + MemChunkGroup memNode, InsertionRequest insertionRequest, InsertRequestContext context) { + List memChunks = new ArrayList<>(); + String tagValue = insertionRequest.getKey(context); + MemChunk child = memNode.get(tagValue); + if (child != null) memChunks.add(child); + return memChunks; + } + + /** + * the insert method corresponding to the MemChunkGroup node + * + * @param memNode memory node + * @param context insert request context + */ + @Override + public void insert( + MemChunkGroup memNode, InsertionRequest insertionRequest, InsertRequestContext context) { + String tagValue = insertionRequest.getKey(context); + memNode.put(tagValue); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java new file mode 100644 index 000000000000..5dd91ef869f3 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemChunkInsertion.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.lsm.annotation.InsertionProcessor; +import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcessor; + +import java.util.List; + +/** insertion for MemChunk */ +@InsertionProcessor(level = 3) +public class MemChunkInsertion extends InsertLevelProcessor { + + /** + * MemChunk is the last layer of memory nodes, no children + * + * @param memNode memory node + * @param context request context + * @return null + */ + @Override + public List getChildren( + MemChunk memNode, InsertionRequest insertionRequest, InsertRequestContext context) { + return null; + } + + /** + * the insert method corresponding to the MemChunk node + * + * @param memNode memory node + * @param context insert request context + */ + @Override + public void insert( + MemChunk memNode, InsertionRequest insertionRequest, InsertRequestContext context) { + Integer deviceID = insertionRequest.getValue(); + memNode.put(deviceID); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java new file mode 100644 index 000000000000..46507443b91f --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableGroupInsertion.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup; +import org.apache.iotdb.lsm.annotation.InsertionProcessor; +import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcessor; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** insertion for MemTableGroup */ +@InsertionProcessor(level = 0) +public class MemTableGroupInsertion + extends InsertLevelProcessor { + + /** + * get all MemTable that need to be processed in the current MemTableGroup + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemTables + */ + @Override + public List getChildren( + MemTableGroup memNode, InsertionRequest request, InsertRequestContext context) { + List memTables = new ArrayList<>(); + memTables.add(memNode.getWorkingMemTable()); + return memTables; + } + + /** + * the insert method corresponding to the MemTableGroup node + * + * @param memNode memory node + * @param context insert request context + */ + @Override + public void insert( + MemTableGroup memNode, InsertionRequest request, InsertRequestContext context) { + int id = request.getValue(); + MemTable workingMemTable = memNode.getWorkingMemTable(); + Map immutableMemTables = memNode.getImmutableMemTables(); + // if the device id can not be saved to the current working MemTable + if (!memNode.inWorkingMemTable(id)) { + workingMemTable.setStatus(MemTable.IMMUTABLE); + immutableMemTables.put( + memNode.getMaxDeviceID() / memNode.getNumOfDeviceIdsInMemTable(), workingMemTable); + memNode.setWorkingMemTable(new MemTable(MemTable.WORKING)); + } + memNode.setMaxDeviceID(id); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java new file mode 100644 index 000000000000..7b20522ae78f --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/insertion/MemTableInsertion.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.insertion; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.annotation.InsertionProcessor; +import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext; +import org.apache.iotdb.lsm.levelProcess.InsertLevelProcessor; + +import java.util.ArrayList; +import java.util.List; + +/** insertion for MemTable */ +@InsertionProcessor(level = 1) +public class MemTableInsertion + extends InsertLevelProcessor { + + /** + * get all MemChunkGroups that need to be processed in the current MemTable + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunkGroups + */ + @Override + public List getChildren( + MemTable memNode, InsertionRequest insertionRequest, InsertRequestContext context) { + if (memNode.isImmutable()) return new ArrayList<>(); + List memChunkGroups = new ArrayList<>(); + String tagKey = insertionRequest.getKey(context); + MemChunkGroup child = memNode.get(tagKey); + if (child != null) memChunkGroups.add(child); + return memChunkGroups; + } + + /** + * the insert method corresponding to the MemTable node + * + * @param memNode memory node + * @param context insert request context + */ + @Override + public void insert( + MemTable memNode, InsertionRequest insertionRequest, InsertRequestContext context) { + if (memNode.isImmutable()) return; + String tagKey = insertionRequest.getKey(context); + memNode.put(tagKey); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java new file mode 100644 index 000000000000..80610c1eaec6 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunk.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import org.roaringbitmap.RoaringBitmap; + +/** used to manage the device id collection */ +public class MemChunk { + + // manage the device id collection, see: https://github.com/RoaringBitmap/RoaringBitmap + private RoaringBitmap roaringBitmap; + + public MemChunk() { + roaringBitmap = new RoaringBitmap(); + } + + public boolean isEmpty() { + if (roaringBitmap == null) return true; + return roaringBitmap.isEmpty(); + } + + @Override + public String toString() { + return roaringBitmap.toString(); + } + + public void put(int id) { + roaringBitmap.add(id); + } + + public void remove(int id) { + roaringBitmap.remove(id); + } + + public RoaringBitmap getRoaringBitmap() { + return this.roaringBitmap; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java new file mode 100644 index 000000000000..c4dca031639a --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemChunkGroup.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import java.util.HashMap; +import java.util.Map; + +/** used to manage tagValue -> MemChunk */ +public class MemChunkGroup { + + // manage tagValue -> MemChunk + private Map memChunkMap; + + public MemChunkGroup() { + memChunkMap = new HashMap<>(); + } + + public void put(String tagValue) { + if (!memChunkMap.containsKey(tagValue)) { + memChunkMap.put(tagValue, new MemChunk()); + } + } + + @Override + public String toString() { + return memChunkMap.toString(); + } + + public MemChunk get(String tagValue) { + return memChunkMap.get(tagValue); + } + + public void remove(String tagValue) { + memChunkMap.remove(tagValue); + } + + public boolean isEmpty() { + return memChunkMap.isEmpty(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java new file mode 100644 index 000000000000..5f9cd09c6284 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTable.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** used to manage tagKey -> MemChunkGroup */ +public class MemTable { + + public static final String WORKING = "working"; + + public static final String IMMUTABLE = "immutable"; + + // manage tagKey -> MemChunkGroup + private Map memChunkGroupMap; + + private String status; + + // if the memTable is immutable, the data cannot be deleted directly, and the deleted data needs + // to be recorded in the deletionList + private Set deletionList; + + public MemTable(String status) { + memChunkGroupMap = new HashMap<>(); + this.status = status; + deletionList = new HashSet<>(); + } + + public void put(String tagKey) { + if (this.status.equals(IMMUTABLE)) return; + if (!memChunkGroupMap.containsKey(tagKey)) { + memChunkGroupMap.put(tagKey, new MemChunkGroup()); + } + } + + @Override + public String toString() { + return "MemTable{" + + "memChunkGroupMap=" + + memChunkGroupMap + + ", status='" + + status + + '\'' + + ", deletionList=" + + deletionList + + '}'; + } + + public MemChunkGroup get(String tagKey) { + return memChunkGroupMap.get(tagKey); + } + + public void remove(String tagKey) { + memChunkGroupMap.remove(tagKey); + } + + public boolean isImmutable() { + return status.equals(IMMUTABLE); + } + + public Set getDeletionList() { + return deletionList; + } + + public void setStatus(String status) { + this.status = status; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTableGroup.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTableGroup.java new file mode 100644 index 000000000000..e49ce546a46f --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/memtable/MemTableGroup.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable; + +import java.util.HashMap; +import java.util.Map; + +/** used to manage working and immutableMemTables */ +public class MemTableGroup { + + // the maximum number of device ids managed by a working memTable + private int numOfDeviceIdsInMemTable; + + // (maxDeviceID / numOfDeviceIdsInMemTable) -> MemTable + private Map immutableMemTables; + + private MemTable workingMemTable; + + // the largest device id saved by the current MemTable + private int maxDeviceID; + + public MemTableGroup() {} + + public MemTableGroup(int numOfDeviceIdsInMemTable) { + this.numOfDeviceIdsInMemTable = numOfDeviceIdsInMemTable; + workingMemTable = new MemTable(MemTable.WORKING); + immutableMemTables = new HashMap<>(); + maxDeviceID = 0; + } + + public int getNumOfDeviceIdsInMemTable() { + return numOfDeviceIdsInMemTable; + } + + public void setNumOfDeviceIdsInMemTable(int numOfDeviceIdsInMemTable) { + this.numOfDeviceIdsInMemTable = numOfDeviceIdsInMemTable; + } + + public Map getImmutableMemTables() { + return immutableMemTables; + } + + public void setImmutableMemTables(Map immutableMemTables) { + this.immutableMemTables = immutableMemTables; + } + + public MemTable getWorkingMemTable() { + return workingMemTable; + } + + public void setWorkingMemTable(MemTable workingMemTable) { + this.workingMemTable = workingMemTable; + } + + public int getMaxDeviceID() { + return maxDeviceID; + } + + public void setMaxDeviceID(int maxDeviceID) { + this.maxDeviceID = maxDeviceID; + } + + /** + * determine whether the id can be saved to the current MemTable + * + * @param id INT32 device id + * @return return true if it can, otherwise return false + */ + public boolean inWorkingMemTable(int id) { + return id / numOfDeviceIdsInMemTable == maxDeviceID / numOfDeviceIdsInMemTable; + } + + @Override + public String toString() { + return "MemTableGroup{" + + "numOfDeviceIdsInMemTable=" + + numOfDeviceIdsInMemTable + + ", immutableMemTables=" + + immutableMemTables + + ", workingMemTable=" + + workingMemTable + + ", maxDeviceID=" + + maxDeviceID + + '}'; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java new file mode 100644 index 000000000000..62636914d58c --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkGroupQuery.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.lsm.annotation.QueryProcessor; +import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcessor; + +import java.util.ArrayList; +import java.util.List; + +/** query for MemChunkGroup */ +@QueryProcessor(level = 2) +public class MemChunkGroupQuery extends QueryLevelProcessor { + + /** + * get all MemChunks that need to be processed in the current MemChunkGroup + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunks + */ + @Override + public List getChildren( + MemChunkGroup memNode, QueryRequest queryRequest, QueryRequestContext context) { + List memChunks = new ArrayList<>(); + String tagValue = queryRequest.getKey(context); + MemChunk child = memNode.get(tagValue); + if (child != null) memChunks.add(child); + return memChunks; + } + + /** + * the query method corresponding to the MemChunkGroup node + * + * @param memNode memory node + * @param context query request context + */ + @Override + public void query( + MemChunkGroup memNode, QueryRequest queryRequest, QueryRequestContext context) {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java new file mode 100644 index 000000000000..0f548033ed07 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemChunkQuery.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunk; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.response.QueryResponse; +import org.apache.iotdb.lsm.annotation.QueryProcessor; +import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcessor; + +import org.roaringbitmap.RoaringBitmap; + +import java.util.List; + +/** query for MemChunk */ +@QueryProcessor(level = 3) +public class MemChunkQuery extends QueryLevelProcessor { + + /** + * MemChunk is the last layer of memory nodes, no children + * + * @param memNode memory node + * @param context request context + * @return null + */ + @Override + public List getChildren( + MemChunk memNode, QueryRequest queryRequest, QueryRequestContext context) { + return null; + } + + /** + * the query method corresponding to the MemChunk node + * + * @param memNode memory node + * @param context query request context + */ + @Override + public void query(MemChunk memNode, QueryRequest queryRequest, QueryRequestContext context) { + QueryResponse response = context.getResponse(); + if (response == null) { + response = new QueryResponse(); + context.setResponse(response); + } + RoaringBitmap roaringBitmap = context.getValue(); + if (roaringBitmap == null) roaringBitmap = new RoaringBitmap(); + RoaringBitmap now = RoaringBitmap.or(roaringBitmap, memNode.getRoaringBitmap()); + context.setValue(now); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java new file mode 100644 index 000000000000..39e09db7f960 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableGroupQuery.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTableGroup; +import org.apache.iotdb.lsm.annotation.QueryProcessor; +import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcessor; + +import java.util.ArrayList; +import java.util.List; + +/** query for MemTableGroup */ +@QueryProcessor(level = 0) +public class MemTableGroupQuery extends QueryLevelProcessor { + + /** + * get all MemTable that need to be processed in the current MemTableGroup + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemTable + */ + @Override + public List getChildren( + MemTableGroup memNode, QueryRequest request, QueryRequestContext context) { + List memTables = new ArrayList<>(); + memTables.add(memNode.getWorkingMemTable()); + memTables.addAll(memNode.getImmutableMemTables().values()); + return memTables; + } + + /** + * the query method corresponding to the MemTable node, do nothing + * + * @param memNode memory node + * @param context query request context + */ + @Override + public void query(MemTableGroup memNode, QueryRequest request, QueryRequestContext context) {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java new file mode 100644 index 000000000000..e08f54cd55c4 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/query/MemTableQuery.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.query; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.QueryRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemChunkGroup; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.memtable.MemTable; +import org.apache.iotdb.lsm.annotation.QueryProcessor; +import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext; +import org.apache.iotdb.lsm.levelProcess.QueryLevelProcessor; + +import org.roaringbitmap.RoaringBitmap; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** query for MemTable */ +@QueryProcessor(level = 1) +public class MemTableQuery extends QueryLevelProcessor { + + /** + * get all MemChunkGroups that need to be processed in the current MemTable + * + * @param memNode memory node + * @param context request context + * @return A list of saved MemChunkGroups + */ + @Override + public List getChildren( + MemTable memNode, QueryRequest queryRequest, QueryRequestContext context) { + List memChunkGroups = new ArrayList<>(); + String tagKey = queryRequest.getKey(context); + MemChunkGroup child = memNode.get(tagKey); + if (child != null) memChunkGroups.add(child); + return memChunkGroups; + } + + /** + * the query method corresponding to the MemTable node + * + * @param memNode memory node + * @param context query request context + */ + @Override + public void query(MemTable memNode, QueryRequest queryRequest, QueryRequestContext context) { + // if the memTable is immutable, we need to delete the id in deletionList in the query result + if (memNode.isImmutable()) { + RoaringBitmap roaringBitmap = context.getValue(); + Set deletionList = memNode.getDeletionList(); + for (Integer id : deletionList) { + roaringBitmap.remove(id); + } + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/response/QueryResponse.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/response/QueryResponse.java new file mode 100644 index 000000000000..1d43d7ae957d --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/response/QueryResponse.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.response; + +import org.apache.iotdb.lsm.response.IResponse; + +import org.roaringbitmap.RoaringBitmap; + +import java.util.ArrayList; +import java.util.List; + +/** Represents a query response */ +public class QueryResponse implements IResponse { + + // response value + private RoaringBitmap roaringBitmap; + + // If an exception needs to be thrown during the processing of the request, this variable can be + // used to accept the exception + private List exceptions; + + @Override + public RoaringBitmap getValue() { + return roaringBitmap; + } + + @Override + public void setValue(RoaringBitmap value) { + this.roaringBitmap = value; + } + + @Override + public List getExceptions() { + return exceptions; + } + + @Override + public void setExceptions(List exceptions) { + this.exceptions = exceptions; + } + + /** + * If an exception needs to be thrown during the processing of the request, this method can be + * used to accept the exception + * + * @param e Exception + */ + @Override + public void addException(Exception e) { + if (exceptions == null) { + exceptions = new ArrayList<>(); + } + exceptions.add(e); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java new file mode 100644 index 000000000000..d3d8176dfb32 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALEntry.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal; + +import org.apache.iotdb.lsm.wal.IWALRecord; +import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** represents a record in the wal file */ +public class WALEntry implements IWALRecord { + + // can be insertion(1) or deletion(2) + private int type; + + // keys at each level + private List keys; + + // device id + private int deviceID; + + public WALEntry() { + super(); + } + + public WALEntry(int type, List keys, int deviceID) { + super(); + this.type = type; + this.keys = keys; + this.deviceID = deviceID; + } + + /** + * serialize the wal entry + * + * @param buffer byte buffer + */ + @Override + public void serialize(ByteBuffer buffer) { + ReadWriteIOUtils.write(type, buffer); + ReadWriteIOUtils.write(deviceID, buffer); + ReadWriteIOUtils.write(keys.size(), buffer); + for (String key : keys) { + ReadWriteIOUtils.write(key, buffer); + } + } + + /** + * deserialize from DataInputStream + * + * @param stream data input stream + * @throws IOException + */ + @Override + public void deserialize(DataInputStream stream) throws IOException { + this.type = stream.readInt(); + this.deviceID = stream.readInt(); + int length = stream.readInt(); + this.keys = new ArrayList<>(); + for (int i = 0; i < length; i++) { + String key = ReadWriteIOUtils.readString(stream); + keys.add(key); + } + } + + /** + * generate wal record using prototyping pattern + * + * @return wal record + */ + @Override + public IWALRecord clone() { + try { + return (IWALRecord) super.clone(); + } catch (CloneNotSupportedException e) { + throw new AssertionError(e.getMessage()); + } + } + + public int getType() { + return type; + } + + public void setType(int type) { + this.type = type; + } + + @Override + public List getKeys() { + return keys; + } + + @Override + public Integer getValue() { + return getDeviceID(); + } + + public void setKeys(List keys) { + this.keys = keys; + } + + public int getDeviceID() { + return deviceID; + } + + public void setDeviceID(int deviceID) { + this.deviceID = deviceID; + } + + @Override + public String toString() { + return "WALEntry{" + "type=" + type + ", keys=" + keys + ", deviceID=" + deviceID + '}'; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java new file mode 100644 index 000000000000..f8c9a1eaec61 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/wal/WALManager.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.wal; + +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.DeletionRequest; +import org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex.Request.InsertionRequest; +import org.apache.iotdb.lsm.request.IRequest; +import org.apache.iotdb.lsm.wal.IWALRecord; +import org.apache.iotdb.lsm.wal.WALReader; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +/** Manage wal entry writes and reads */ +public class WALManager extends org.apache.iotdb.lsm.manager.WALManager { + + private static final Logger logger = LoggerFactory.getLogger(WALManager.class); + + private static final int INSERT = 1; + + private static final int DELETE = 2; + + public WALManager( + String schemaDirPath, + String walFileName, + int walBufferSize, + IWALRecord walRecord, + boolean forceEachWrite) + throws IOException { + super(schemaDirPath, walFileName, walBufferSize, walRecord, forceEachWrite); + } + + public WALManager(String schemaDirPath) { + super(schemaDirPath); + } + + /** + * handle wal log writes for each request context + * + * @param request request context + * @throws IOException + */ + @Override + public synchronized void write(IRequest request) { + if (isRecover()) return; + try { + switch (request.getRequestType()) { + case INSERT: + process((InsertionRequest) request); + break; + case DELETE: + process((DeletionRequest) request); + break; + default: + break; + } + } catch (IOException e) { + logger.error(e.getMessage()); + } + } + + /** + * for recover + * + * @return request context + */ + @Override + public synchronized IRequest read() { + WALReader walReader = getWalReader(); + if (walReader.hasNext()) { + WALEntry walEntry = (WALEntry) getWalReader().next(); + if (walEntry.getType() == INSERT) { + return generateInsertRequest(walEntry); + } + if (walEntry.getType() == DELETE) { + return generateDeleteContext(walEntry); + } + } + return null; + } + + /** + * generate insert context from wal entry + * + * @param walEntry wal entry + * @return insert context + */ + private InsertionRequest generateInsertRequest(WALEntry walEntry) { + return new InsertionRequest(walEntry.getKeys(), walEntry.getDeviceID()); + } + + /** + * generate delete context from wal entry + * + * @param walEntry wal entry + * @return delete context + */ + private DeletionRequest generateDeleteContext(WALEntry walEntry) { + return new DeletionRequest(walEntry.getKeys(), walEntry.getDeviceID()); + } + + /** + * handle wal log writes for each insert context + * + * @param request insert request + * @throws IOException + */ + private void process(InsertionRequest request) throws IOException { + WALEntry walEntry = new WALEntry(INSERT, request.getKeys(), request.getValue()); + getWalWriter().write(walEntry); + } + + /** + * handle wal log writes for each delete context + * + * @param request delete context + * @throws IOException + */ + private void process(DeletionRequest request) throws IOException { + WALEntry walEntry = new WALEntry(DELETE, request.getKeys(), request.getValue()); + getWalWriter().write(walEntry); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java new file mode 100644 index 000000000000..d83c016ee0c6 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/MeasurementPathUtils.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.metadata.idtable.entry.DiskSchemaEntry; +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; +import org.apache.iotdb.tsfile.write.schema.MeasurementSchema; + +/** process MeasurementPath */ +public class MeasurementPathUtils { + + /** + * generate MeasurementPath + * + * @param devicePath device path + * @param measurement measurement + * @param schemaEntry schema entry + * @param isAligned is aligned + * @return MeasurementPath + * @throws IllegalPathException + */ + public static MeasurementPath generateMeasurementPath( + String devicePath, String measurement, SchemaEntry schemaEntry, boolean isAligned) + throws IllegalPathException { + MeasurementPath measurementPath = + new MeasurementPath( + devicePath, + measurement, + new MeasurementSchema( + measurement, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType())); + measurementPath.setUnderAlignedEntity(isAligned); + return measurementPath; + } + + /** + * generate MeasurementPath + * + * @param diskSchemaEntry disk schema entry + * @return MeasurementPath + * @throws IllegalPathException + */ + public static MeasurementPath generateMeasurementPath(DiskSchemaEntry diskSchemaEntry) + throws IllegalPathException { + MeasurementPath measurementPath = + new MeasurementPath( + new PartialPath(diskSchemaEntry.seriesKey), + new MeasurementSchema( + diskSchemaEntry.measurementName, + TSDataType.deserialize(diskSchemaEntry.type), + TSEncoding.deserialize(diskSchemaEntry.encoding), + CompressionType.deserialize(diskSchemaEntry.compressor))); + measurementPath.setUnderAlignedEntity(diskSchemaEntry.isAligned); + return measurementPath; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java new file mode 100644 index 000000000000..59208fc418b6 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/PathTagConverterUtils.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; + +import java.util.Map; +import java.util.TreeMap; + +/** path and tag converter */ +public class PathTagConverterUtils { + + /** + * convert the path of the tree model to the tags of the tag model + * + * @param storageGroupFullPath storage group full path + * @param path path of the tree model + * @return tags of the tag model + */ + public static Map pathToTags(String storageGroupFullPath, String path) { + if (path.length() <= storageGroupFullPath.length()) return new TreeMap<>(); + String devicePath = path.substring(storageGroupFullPath.length() + 1); + String[] tags = devicePath.split("\\."); + Map tagsMap = new TreeMap<>(); + for (int i = 0; i < tags.length; i += 2) { + tagsMap.put(tags[i], tags[i + 1]); + } + return tagsMap; + } + + /** + * convert the tags of the tag model to the path of the tree model + * + * @param storageGroupFullPath storage group full path + * @param tags tags of the tag model + * @return path of the tree model + */ + public static String tagsToPath(String storageGroupFullPath, Map tags) { + StringBuilder stringBuilder = new StringBuilder(storageGroupFullPath); + for (String tagKey : tags.keySet()) { + stringBuilder.append(".").append(tagKey).append(".").append(tags.get(tagKey)); + } + return stringBuilder.toString(); + } + + /** + * generate unique path for paths with the same semantics + * + * @param storageGroupFullPath storage group full path + * @param path path of the tree model + * @return unique path of the tree model + */ + public static String pathToTagsSortPath(String storageGroupFullPath, String path) { + return tagsToPath(storageGroupFullPath, pathToTags(storageGroupFullPath, path)); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java new file mode 100644 index 000000000000..18d4c2266219 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/db/metadata/tagSchemaRegion/utils/ShowTimeSeriesResultUtils.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.utils; + +import org.apache.iotdb.db.metadata.idtable.entry.SchemaEntry; +import org.apache.iotdb.db.query.dataset.ShowTimeSeriesResult; + +import java.util.HashMap; + +/** process show timeSeries result */ +public class ShowTimeSeriesResultUtils { + + /** + * generate show timeSeries result + * + * @param sgName storage group name + * @param devicePath device path + * @param measurement measurement + * @param schemaEntry schema entry + * @return ShowTimeSeriesResult + */ + public static ShowTimeSeriesResult generateShowTimeSeriesResult( + String sgName, String devicePath, String measurement, SchemaEntry schemaEntry) { + return new ShowTimeSeriesResult( + devicePath + "." + measurement, + null, + sgName, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>()); + } + + /** + * generate show timeSeries result + * + * @param sgName storage group name + * @param timeSeriesPath timeSeries path + * @param schemaEntry schema entry + * @return ShowTimeSeriesResult + */ + public static ShowTimeSeriesResult generateShowTimeSeriesResult( + String sgName, String timeSeriesPath, SchemaEntry schemaEntry) { + return new ShowTimeSeriesResult( + timeSeriesPath, + null, + sgName, + schemaEntry.getTSDataType(), + schemaEntry.getTSEncoding(), + schemaEntry.getCompressionType(), + schemaEntry.getLastTime(), + new HashMap<>(), + new HashMap<>()); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcessor.java new file mode 100644 index 000000000000..e4d1d6e8b228 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/DeletionProcessor.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Indicates that the class using this annotation is a DeleteLevelProcessor, and the attribute level + * of the annotation indicates which layer of memory node the deletion method of the + * DeleteLevelProcessor is. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface DeletionProcessor { + // level of the DeleteLevelProcessor + int level() default -1; +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcessor.java new file mode 100644 index 000000000000..c06a085eed81 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/InsertionProcessor.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Indicates that the class using this annotation is a InsertLevelProcessor, and the attribute level + * of the annotation indicates which layer of memory node the insertion method of the + * InsertLevelProcessor is. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface InsertionProcessor { + // level of the InsertLevelProcessor + int level() default -1; +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcessor.java new file mode 100644 index 000000000000..f7734b68989a --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/annotation/QueryProcessor.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Indicates that the class using this annotation is a QueryLevelProcessor, and the attribute level + * of the annotation indicates which layer of memory node the query method of the + * QueryLevelProcessor is. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface QueryProcessor { + // level of the QueryLevelProcessor + int level() default -1; +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/applicationcontext/ApplicationContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/applicationcontext/ApplicationContext.java new file mode 100644 index 000000000000..8a7f6646147f --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/applicationcontext/ApplicationContext.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context.applicationcontext; + +import java.util.List; + +/** + * Indicates the information of a user program, the user can customize the configuration startup + * information, and the lsm framework can use this information to generate the lsm engine + */ +public class ApplicationContext { + + // Save the insertion level processor of each layer in hierarchical order + List insertionLevelProcessClass; + + // Save the deletion level processor of each layer in hierarchical order + List deletionLevelProcessClass; + + // Save the query level processor of each layer in hierarchical order + List queryLevelProcessClass; + + public List getInsertionLevelProcessClass() { + return insertionLevelProcessClass; + } + + public void setInsertionLevelProcessClass(List insertionLevelProcessClass) { + this.insertionLevelProcessClass = insertionLevelProcessClass; + } + + public List getDeletionLevelProcessClass() { + return deletionLevelProcessClass; + } + + public void setDeletionLevelProcessClass(List deletionLevelProcessClass) { + this.deletionLevelProcessClass = deletionLevelProcessClass; + } + + public List getQueryLevelProcessClass() { + return queryLevelProcessClass; + } + + public void setQueryLevelProcessClass(List queryLevelProcessClass) { + this.queryLevelProcessClass = queryLevelProcessClass; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/applicationcontext/ApplicationContextGenerator.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/applicationcontext/ApplicationContextGenerator.java new file mode 100644 index 000000000000..c1bbfc557160 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/applicationcontext/ApplicationContextGenerator.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context.applicationcontext; + +import org.apache.iotdb.lsm.annotation.DeletionProcessor; +import org.apache.iotdb.lsm.annotation.InsertionProcessor; +import org.apache.iotdb.lsm.annotation.QueryProcessor; + +import org.reflections.Reflections; +import org.reflections.util.ConfigurationBuilder; +import org.reflections.util.FilterBuilder; + +import java.lang.annotation.Annotation; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** Used to generate ApplicationContext object based on annotations or configuration files */ +public class ApplicationContextGenerator { + + /** + * Scan the package to get all classes, and generate ApplicationContext object based on the + * annotations of these classes + * + * @param packageName package name + * @return ApplicationContext object + */ + public static ApplicationContext GeneratePropertyWithAnnotation(String packageName) { + Reflections reflections = + new Reflections( + new ConfigurationBuilder() + .forPackage(packageName) + .filterInputsBy(new FilterBuilder().includePackage(packageName))); + ApplicationContext applicationContext = new ApplicationContext(); + setDeletionLevelProcessor(applicationContext, reflections); + setInsertionLevelProcessor(applicationContext, reflections); + setQueryLevelProcessor(applicationContext, reflections); + return applicationContext; + } + + /** + * Assign value to the insertion level processor of the ApplicationContext object + * + * @param applicationContext ApplicationContext object + * @param reflections This object holds all the classes scanned in the package + */ + private static void setInsertionLevelProcessor( + ApplicationContext applicationContext, Reflections reflections) { + Set> annotated = reflections.getTypesAnnotatedWith(InsertionProcessor.class); + List levelProcessClass = new ArrayList<>(); + for (Class clz : annotated) { + InsertionProcessor annotationInfo = clz.getAnnotation(InsertionProcessor.class); + setLevelProcessors(levelProcessClass, clz, annotationInfo.level()); + } + applicationContext.setInsertionLevelProcessClass(levelProcessClass); + } + + /** + * Assign value to the deletion level processor of the ApplicationContext object + * + * @param applicationContext ApplicationContext object + * @param reflections This object holds all the classes scanned in the package + */ + private static void setDeletionLevelProcessor( + ApplicationContext applicationContext, Reflections reflections) { + Set> annotated = reflections.getTypesAnnotatedWith(DeletionProcessor.class); + List levelProcessClass = new ArrayList<>(); + for (Class clz : annotated) { + DeletionProcessor annotationInfo = clz.getAnnotation(DeletionProcessor.class); + setLevelProcessors(levelProcessClass, clz, annotationInfo.level()); + } + applicationContext.setDeletionLevelProcessClass(levelProcessClass); + } + + /** + * Assign value to the query level processor of the ApplicationContext object + * + * @param applicationContext ApplicationContext object + * @param reflections This object holds all the classes scanned in the package + */ + private static void setQueryLevelProcessor( + ApplicationContext applicationContext, Reflections reflections) { + List levelProcessClass = new ArrayList<>(); + Set> annotated = reflections.getTypesAnnotatedWith(QueryProcessor.class); + for (Class clz : annotated) { + QueryProcessor annotationInfo = clz.getAnnotation(QueryProcessor.class); + setLevelProcessors(levelProcessClass, clz, annotationInfo.level()); + } + applicationContext.setQueryLevelProcessClass(levelProcessClass); + } + + private static void setLevelProcessors( + List levelProcessorClass, Class clz, int level) { + if (level < levelProcessorClass.size()) { + levelProcessorClass.set(level, clz.getName()); + } else { + for (int i = levelProcessorClass.size(); i < level; i++) { + levelProcessorClass.add(""); + } + levelProcessorClass.add(clz.getName()); + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/DeleteRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/DeleteRequestContext.java new file mode 100644 index 000000000000..1d996680b1c9 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/DeleteRequestContext.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context.requestcontext; + +import org.apache.iotdb.lsm.strategy.PostOrderAccessStrategy; + +/** + * represents the context of a deletion request, this class can be extended to implement a custom + * context + */ +public class DeleteRequestContext extends RequestContext { + + public DeleteRequestContext() { + super(); + // post-order traversal strategy is used by default + accessStrategy = new PostOrderAccessStrategy(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/FlushRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/FlushRequestContext.java new file mode 100644 index 000000000000..29e0ed1b9c96 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/FlushRequestContext.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context.requestcontext; + +import org.apache.iotdb.lsm.strategy.RBFSAccessStrategy; + +/** + * represents the context of a flush request, this class can be extended to implement a custom + * context + */ +public class FlushRequestContext extends RequestContext { + public FlushRequestContext() { + super(); + // use the reverse breadth-first traversal strategy to access memory nodes + accessStrategy = new RBFSAccessStrategy(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/InsertRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/InsertRequestContext.java new file mode 100644 index 000000000000..d519f6994e90 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/InsertRequestContext.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context.requestcontext; + +import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy; + +/** + * represents the context of a insertion request, this class can be extended to implement a custom + * context + */ +public class InsertRequestContext extends RequestContext { + + public InsertRequestContext() { + super(); + // preorder traversal strategy is used by default + accessStrategy = new PreOrderAccessStrategy(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/QueryRequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/QueryRequestContext.java new file mode 100644 index 000000000000..7534dee516fb --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/QueryRequestContext.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context.requestcontext; + +import org.apache.iotdb.lsm.strategy.PostOrderAccessStrategy; + +/** + * represents the context of a query request, this class can be extended to implement a custom + * context + */ +public class QueryRequestContext extends RequestContext { + + public QueryRequestContext() { + super(); + // post-order traversal strategy is used by default + accessStrategy = new PostOrderAccessStrategy(); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/RequestContext.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/RequestContext.java new file mode 100644 index 000000000000..626d60579741 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/context/requestcontext/RequestContext.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.context.requestcontext; + +import org.apache.iotdb.lsm.response.IResponse; +import org.apache.iotdb.lsm.strategy.IAccessStrategy; +import org.apache.iotdb.lsm.strategy.PreOrderAccessStrategy; + +/** represents the context of a request */ +public class RequestContext { + + // memory Structure Access Policy + IAccessStrategy accessStrategy; + + // the tree level of the currently pending memory node + int level; + + // the maximum level of memory nodes that can be processed + int levelUpperBound; + + // response, encapsulating the response value and exception information. + IResponse response; + + public RequestContext() { + // preorder traversal strategy is used by default + accessStrategy = new PreOrderAccessStrategy(); + level = 0; + levelUpperBound = Integer.MAX_VALUE; + } + + public void setLevel(int level) { + this.level = level; + } + + public int getLevel() { + return level; + } + + public IAccessStrategy getAccessStrategy() { + return accessStrategy; + } + + public void setAccessStrategy(IAccessStrategy accessStrategy) { + this.accessStrategy = accessStrategy; + } + + public int getLevelUpperBound() { + return levelUpperBound; + } + + public void setLevelUpperBound(int levelUpperBound) { + this.levelUpperBound = levelUpperBound; + } + + public R getResponse() { + return (R) response; + } + + public void setResponse(R response) { + this.response = response; + } + + /** + * get the result of the response + * + * @param type of the result + * @return response result + */ + public T getValue() { + return (T) getResponse().getValue(); + } + + /** + * set the result of the response + * + * @param value response result + * @param type of the response result + */ + public void setValue(T value) { + response.setValue(value); + } + + /** + * If an exception needs to be thrown during the processing of the request, this method can be + * used to accept the exception + * + * @param e Exception + */ + public void thrownException(Exception e) { + response.addException(e); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java new file mode 100644 index 000000000000..e6c2530c47b8 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/ILSMEngine.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.engine; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.lsm.request.IDeletionRequest; +import org.apache.iotdb.lsm.request.IInsertionRequest; +import org.apache.iotdb.lsm.request.IQueryRequest; +import org.apache.iotdb.lsm.response.IResponse; + +import java.io.IOException; + +/** + * This interface defines the appearance of the LSM framework and provides read and write methods + */ +public interface ILSMEngine extends IRecoverable { + + /** + * Use this ILSMEngine to insert data + * + * @param insertionRequest Encapsulates the data to be inserted + * @param The type of key in the request data + * @param The type of value in the request data + * @param type of response + */ + R insert(IInsertionRequest insertionRequest); + + /** + * Use this ILSMEngine to query + * + * @param queryRequest Encapsulates query data + * @param The type of key in the request data + * @param type of response + */ + R query(IQueryRequest queryRequest); + + /** + * Use this ILSMEngine to delete data + * + * @param deletionRequest Encapsulates the data to be deleted + * @param The type of key in the request data + * @param The type of value in the request data + * @param type of response + */ + R delete(IDeletionRequest deletionRequest); + + /** recover the ILSMEngine */ + void recover(); + + /** + * Close all open resources + * + * @throws IOException + */ + @TestOnly + void clear() throws IOException; +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/IRecoverable.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/IRecoverable.java new file mode 100644 index 000000000000..d5fe23dda70f --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/IRecoverable.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.engine; + +import org.apache.iotdb.lsm.request.IRequest; + +/** Any object implements this interface can be recovered by RecoverManager */ +public interface IRecoverable { + + /** + * Use Request to recover the object which implements the interface + * + * @param request insertionRequest or deletionRequest + * @param The type of key in the request data + * @param The type of value in the request data + */ + void recover(IRequest request); +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java new file mode 100644 index 000000000000..97420c173c9d --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngine.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.engine; + +import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext; +import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext; +import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext; +import org.apache.iotdb.lsm.manager.DeletionManager; +import org.apache.iotdb.lsm.manager.InsertionManager; +import org.apache.iotdb.lsm.manager.QueryManager; +import org.apache.iotdb.lsm.manager.RecoverManager; +import org.apache.iotdb.lsm.manager.WALManager; +import org.apache.iotdb.lsm.request.IDeletionRequest; +import org.apache.iotdb.lsm.request.IInsertionRequest; +import org.apache.iotdb.lsm.request.IQueryRequest; +import org.apache.iotdb.lsm.request.IRequest; +import org.apache.iotdb.lsm.response.IResponse; + +import java.io.IOException; + +/** + * The default ILSMEngine implementation class provided by the LSM framework + * + * @param The type of root memory node handled by this engine + */ +public class LSMEngine implements ILSMEngine { + + // Use the framework's default InsertionManager object to handle insert requests + private InsertionManager insertionManager; + + // Use the framework's default DeletionManager object to handle delete requests + private DeletionManager deletionManager; + + // Use the framework's default QueryManager object to handle query requests + private QueryManager queryManager; + + // Used to manage wal logs + private WALManager walManager; + + // Use the framework's default RecoverManager object to recover the LSMEngine + private RecoverManager> recoverManager; + + // Managed root memory node + private T rootMemNode; + + public LSMEngine() {} + + /** + * Use this LSMEngine to insert data + * + * @param insertionRequest Encapsulates the data to be inserted + * @param The type of key in the request data + * @param The type of value in the request data + * @param type of response + */ + @Override + public R insert(IInsertionRequest insertionRequest) { + InsertRequestContext insertRequestContext = new InsertRequestContext(); + insertionManager.process(rootMemNode, insertionRequest, insertRequestContext); + return insertRequestContext.getResponse(); + } + + /** + * Use this LSMEngine to query + * + * @param queryRequest Encapsulates query data + * @param The type of key in the request data + * @param type of response + */ + @Override + public R query(IQueryRequest queryRequest) { + QueryRequestContext queryRequestContext = new QueryRequestContext(); + queryManager.process(rootMemNode, queryRequest, queryRequestContext); + return queryRequestContext.getResponse(); + } + + /** + * Use this LSMEngine to delete data + * + * @param deletionRequest Encapsulates the data to be deleted + * @param The type of key in the request data + * @param The type of value in the request data + * @param type of response + */ + @Override + public R delete(IDeletionRequest deletionRequest) { + DeleteRequestContext deleteRequestContext = new DeleteRequestContext(); + deletionManager.process(rootMemNode, deletionRequest, deleteRequestContext); + return deleteRequestContext.getResponse(); + } + + /** recover the LSMEngine */ + @Override + public void recover() { + recoverManager.recover(this); + } + + /** + * Close all open resources + * + * @throws IOException + */ + @Override + @TestOnly + public void clear() throws IOException { + walManager.close(); + } + + /** + * Use Request to recover the LSMEngine + * + * @param request insertionRequest or deletionRequest + * @param The type of key in the request data + * @param The type of value in the request data + */ + @Override + public void recover(IRequest request) { + switch (request.getRequestType()) { + case INSERT: + insert((IInsertionRequest) request); + break; + case DELETE: + delete((IDeletionRequest) request); + break; + default: + break; + } + } + + protected void setInsertionManager( + InsertionManager insertionManager) { + this.insertionManager = (InsertionManager) insertionManager; + } + + protected void setDeletionManager( + DeletionManager deletionManager) { + this.deletionManager = (DeletionManager) deletionManager; + } + + protected void setQueryManager(QueryManager queryManager) { + this.queryManager = (QueryManager) queryManager; + } + + protected WALManager getWalManager() { + return walManager; + } + + protected void setWalManager(WALManager walManager) { + this.walManager = walManager; + } + + protected void setRecoverManager(RecoverManager> recoverManager) { + this.recoverManager = recoverManager; + } + + protected void setRootMemNode(T rootMemNode) { + this.rootMemNode = rootMemNode; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java new file mode 100644 index 000000000000..580575277331 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/engine/LSMEngineBuilder.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.engine; + +import org.apache.iotdb.lsm.context.applicationcontext.ApplicationContext; +import org.apache.iotdb.lsm.context.applicationcontext.ApplicationContextGenerator; +import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext; +import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext; +import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext; +import org.apache.iotdb.lsm.context.requestcontext.RequestContext; +import org.apache.iotdb.lsm.levelProcess.ILevelProcessor; +import org.apache.iotdb.lsm.levelProcess.LevelProcessorChain; +import org.apache.iotdb.lsm.manager.DeletionManager; +import org.apache.iotdb.lsm.manager.InsertionManager; +import org.apache.iotdb.lsm.manager.QueryManager; +import org.apache.iotdb.lsm.manager.RecoverManager; +import org.apache.iotdb.lsm.manager.WALManager; +import org.apache.iotdb.lsm.request.IDeletionRequest; +import org.apache.iotdb.lsm.request.IInsertionRequest; +import org.apache.iotdb.lsm.request.IQueryRequest; +import org.apache.iotdb.lsm.request.IRequest; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.util.List; + +/** + * Build the LSMEngine object + * + * @param The type of root memory node handled by this engine + */ +public class LSMEngineBuilder { + + private static final Logger logger = LoggerFactory.getLogger(LSMEngineBuilder.class); + + // The constructed LSMEngine object + private LSMEngine lsmEngine; + + public LSMEngineBuilder() { + lsmEngine = new LSMEngine<>(); + } + + /** + * build WalManager for lsmEngine + * + * @param walManager WalManager object + */ + public LSMEngineBuilder buildWalManager(WALManager walManager) { + lsmEngine.setWalManager(walManager); + return this; + } + + /** + * build InsertionManager for lsmEngine + * + * @param levelProcessChain insert level processors chain + * @param extends IInsertionRequest + */ + public LSMEngineBuilder buildInsertionManager( + LevelProcessorChain levelProcessChain) { + InsertionManager insertionManager = new InsertionManager<>(lsmEngine.getWalManager()); + insertionManager.setLevelProcessorsChain(levelProcessChain); + buildInsertionManager(insertionManager); + return this; + } + + /** + * build InsertionManager for lsmEngine + * + * @param insertionManager InsertionManager object + * @param extends IInsertionRequest + */ + public LSMEngineBuilder buildInsertionManager( + InsertionManager insertionManager) { + lsmEngine.setInsertionManager(insertionManager); + return this; + } + + /** + * build DeletionManager for lsmEngine + * + * @param levelProcessChain delete level processors chain + * @param extends IDeletionRequest + */ + public LSMEngineBuilder buildDeletionManager( + LevelProcessorChain levelProcessChain) { + DeletionManager deletionManager = new DeletionManager<>(lsmEngine.getWalManager()); + deletionManager.setLevelProcessorsChain(levelProcessChain); + buildDeletionManager(deletionManager); + return this; + } + + /** + * build DeletionManager for lsmEngine + * + * @param deletionManager DeletionManager object + * @param extends IDeletionRequest + */ + public LSMEngineBuilder buildDeletionManager( + DeletionManager deletionManager) { + lsmEngine.setDeletionManager(deletionManager); + return this; + } + + /** + * build QueryManager for lsmEngine + * + * @param levelProcessChain query level processors chain + * @param extends IQueryRequest + */ + public LSMEngineBuilder buildQueryManager( + LevelProcessorChain levelProcessChain) { + QueryManager queryManager = new QueryManager<>(); + queryManager.setLevelProcessorsChain(levelProcessChain); + buildQueryManager(queryManager); + return this; + } + + /** + * build QueryManager for lsmEngine + * + * @param queryManager QueryManager object + * @param extends IQueryRequest + */ + public LSMEngineBuilder buildQueryManager( + QueryManager queryManager) { + lsmEngine.setQueryManager(queryManager); + return this; + } + + /** build RecoverManager for lsmEngine */ + public LSMEngineBuilder buildRecoverManager() { + RecoverManager> recoverManager = new RecoverManager<>(lsmEngine.getWalManager()); + lsmEngine.setRecoverManager(recoverManager); + return this; + } + + /** + * build root memory node for lsmEngine + * + * @param rootMemNode root memory node + */ + public LSMEngineBuilder buildRootMemNode(T rootMemNode) { + lsmEngine.setRootMemNode(rootMemNode); + return this; + } + + /** + * build level processors from ApplicationContext object + * + * @param applicationContext ApplicationContext object + */ + private LSMEngineBuilder buildLevelProcessors(ApplicationContext applicationContext) { + LevelProcessorChain insertionLevelProcessChain = + generateLevelProcessorsChain(applicationContext.getInsertionLevelProcessClass()); + LevelProcessorChain deletionLevelProcessChain = + generateLevelProcessorsChain(applicationContext.getDeletionLevelProcessClass()); + LevelProcessorChain queryLevelProcessChain = + generateLevelProcessorsChain(applicationContext.getQueryLevelProcessClass()); + return buildQueryManager(queryLevelProcessChain) + .buildInsertionManager(insertionLevelProcessChain) + .buildDeletionManager(deletionLevelProcessChain); + } + + /** + * Scan the classes of the package and build level processors based on the class annotations + * + * @param packageName package name + */ + private LSMEngineBuilder buildLevelProcessors(String packageName) { + try { + ApplicationContext property = + ApplicationContextGenerator.GeneratePropertyWithAnnotation(packageName); + buildLevelProcessors(property); + } catch (Exception e) { + logger.error(e.getMessage()); + } + return this; + } + + /** + * build all LSM managers + * + * @param applicationContext ApplicationContext object + * @param walManager WalManager object + */ + public LSMEngineBuilder buildLSMManagers( + ApplicationContext applicationContext, WALManager walManager) { + try { + buildWalManager(walManager).buildLevelProcessors(applicationContext).buildRecoverManager(); + } catch (Exception e) { + logger.error(e.getMessage()); + } + return this; + } + + /** + * Scan the classes of the package and build all LSM managers based on the class annotations + * + * @param packageName package name + * @param walManager WalManager object + */ + public LSMEngineBuilder buildLSMManagers(String packageName, WALManager walManager) { + try { + ApplicationContext property = + ApplicationContextGenerator.GeneratePropertyWithAnnotation(packageName); + buildLSMManagers(property, walManager); + } catch (Exception e) { + logger.error(e.getMessage()); + } + return this; + } + + /** + * Get the built lsmEngine + * + * @return LSMEngine object + */ + public LSMEngine build() { + return lsmEngine; + } + + /** + * generate level processors Chain + * + * @param levelProcessorClassNames Save all level processor class names in hierarchical order + * @param extends IRequest + * @param extends RequestContext + * @return level Processors Chain + */ + private + LevelProcessorChain generateLevelProcessorsChain( + List levelProcessorClassNames) { + LevelProcessorChain levelProcessChain = new LevelProcessorChain<>(); + try { + if (levelProcessorClassNames.size() > 0) { + ILevelProcessor iLevelProcess = + levelProcessChain.nextLevel(generateLevelProcessor(levelProcessorClassNames.get(0))); + for (int i = 1; i < levelProcessorClassNames.size(); i++) { + iLevelProcess = + iLevelProcess.nextLevel(generateLevelProcessor(levelProcessorClassNames.get(i))); + } + } + } catch (Exception e) { + logger.error(e.getMessage()); + } + return levelProcessChain; + } + + /** + * generate level processor + * + * @param className level processor class name + * @param extends IRequest + * @param extends RequestContext + * @return level processor + * @throws ClassNotFoundException + * @throws NoSuchMethodException + * @throws InvocationTargetException + * @throws InstantiationException + * @throws IllegalAccessException + */ + private + ILevelProcessor generateLevelProcessor(String className) + throws ClassNotFoundException, NoSuchMethodException, InvocationTargetException, + InstantiationException, IllegalAccessException { + Class c = Class.forName(className); + ILevelProcessor result = + (ILevelProcessor) c.getDeclaredConstructor().newInstance(); + return result; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcessor.java new file mode 100644 index 000000000000..e32aad940476 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/BasicLevelProcessor.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.requestcontext.RequestContext; + +import java.util.List; + +/** the processing method corresponding to each layer of memory nodes */ +public abstract class BasicLevelProcessor + implements ILevelProcessor { + + // the next level process + ILevelProcessor next; + + /** + * process the current layer memory node + * + * @param memNode memory node + * @param context request context + */ + public abstract void handle(I memNode, R request, C context); + + /** + * get the memory node that needs to be processed in the next layer + * + * @param memNode memory node + * @param context request context + * @return all next-level memory nodes that need to be processed + */ + public abstract List getChildren(I memNode, R request, C context); + + /** + * add the LevelProcess of the next layer of memory nodes + * + * @param next LevelProcess of the next layer + * @return the next level process + */ + @Override + public ILevelProcessor nextLevel(ILevelProcessor next) { + this.next = next; + return next; + } + + /** + * use this method to process memory nodes at each layer according to the access strategy + * + * @param memNode memory node + * @param context request context + */ + @Override + public void process(I memNode, R request, C context) { + context.getAccessStrategy().execute(this, memNode, request, context); + } + + public boolean hasNext() { + return next != null; + } + + public ILevelProcessor getNext() { + return next; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcessor.java new file mode 100644 index 000000000000..e46da38fee53 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/DeleteLevelProcessor.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext; + +/** indicates the deletion method of each layer of memory nodes */ +public abstract class DeleteLevelProcessor + extends BasicLevelProcessor { + + /** + * the deletion method of memory node + * + * @param memNode memory node + * @param context deletion request context + */ + public abstract void delete(I memNode, R request, DeleteRequestContext context); + + @Override + public void handle(I memNode, R request, DeleteRequestContext context) { + delete(memNode, request, context); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcessor.java new file mode 100644 index 000000000000..ca92c575174e --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/FlushLevelProcessor.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.requestcontext.FlushRequestContext; + +/** indicates the flush method of each layer of memory nodes */ +public abstract class FlushLevelProcessor + extends BasicLevelProcessor { + + /** + * the flush method of memory node + * + * @param memNode memory node + * @param context flush request context + */ + public abstract void flush(I memNode, FlushRequestContext context); + + public void handle(I memNode, Object request, FlushRequestContext context) { + flush(memNode, context); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcessor.java new file mode 100644 index 000000000000..871fd180d417 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/ILevelProcessor.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.requestcontext.RequestContext; + +/** the processing method corresponding to each layer of memory nodes */ +public interface ILevelProcessor { + + /** + * add the LevelProcess of the next layer of memory nodes + * + * @param next LevelProcess of the next layer + * @return LevelProcess of the next layer + */ + ILevelProcessor nextLevel(ILevelProcessor next); + + /** + * use this method to process memory nodes at each layer according to the access strategy + * + * @param memNode memory node + * @param context request context + */ + void process(I memNode, R request, C context); +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcessor.java new file mode 100644 index 000000000000..ae1240f75c08 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/InsertLevelProcessor.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext; + +/** indicates the insertion method of each layer of memory nodes */ +public abstract class InsertLevelProcessor + extends BasicLevelProcessor { + + /** + * the insertion method of memory node + * + * @param memNode memory node + * @param context insertion request context + */ + public abstract void insert(I memNode, R request, InsertRequestContext context); + + @Override + public void handle(I memNode, R request, InsertRequestContext context) { + insert(memNode, request, context); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessorChain.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessorChain.java new file mode 100644 index 000000000000..f1e8b9aed391 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/LevelProcessorChain.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.requestcontext.RequestContext; +import org.apache.iotdb.lsm.request.IRequest; + +/** Save the level processor of each layer in hierarchical order */ +public class LevelProcessorChain { + + // the level process of the first layer of memory nodes + ILevelProcessor headLevelProcess; + + public ILevelProcessor nextLevel(ILevelProcessor next) { + this.headLevelProcess = next; + return next; + } + + /** + * Use the level processor of each layer to process memory nodes + * + * @param memNode memory node + * @param request extends IRequest + * @param context extends RequestContext + */ + public void process(T memNode, R request, C context) { + headLevelProcess.process(memNode, request, context); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcessor.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcessor.java new file mode 100644 index 000000000000..1caa24844a3f --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/levelProcess/QueryLevelProcessor.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.levelProcess; + +import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext; + +/** indicates the query method of each layer of memory nodes */ +public abstract class QueryLevelProcessor + extends BasicLevelProcessor { + + /** + * the query method of memory node + * + * @param memNode memory node + * @param context query request context + */ + public abstract void query(I memNode, R request, QueryRequestContext context); + + @Override + public void handle(I memNode, R request, QueryRequestContext context) { + query(memNode, request, context); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java new file mode 100644 index 000000000000..a4a3eec8486f --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/BasicLSMManager.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.manager; + +import org.apache.iotdb.lsm.context.requestcontext.RequestContext; +import org.apache.iotdb.lsm.levelProcess.LevelProcessorChain; +import org.apache.iotdb.lsm.request.IRequest; + +/** basic lsm manager implementation */ +public abstract class BasicLSMManager + implements ILSMManager { + + // the level process of the first layer of memory nodes + LevelProcessorChain levelProcessChain; + + public BasicLSMManager() {} + + /** + * processing of the root memory node + * + * @param root root memory node + * @param context request context + */ + @Override + public void process(T root, R request, C context) { + preProcess(root, request, context); + levelProcessChain.process(root, request, context); + postProcess(root, request, context); + } + + /** + * set level processors chain + * + * @param levelProcessorsChain level processors chain + */ + public void setLevelProcessorsChain(LevelProcessorChain levelProcessorsChain) { + this.levelProcessChain = levelProcessorsChain; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java new file mode 100644 index 000000000000..93c632ce2c77 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/DeletionManager.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.manager; + +import org.apache.iotdb.lsm.context.requestcontext.DeleteRequestContext; +import org.apache.iotdb.lsm.request.IDeletionRequest; + +/** manage deletion to root memory node */ +public class DeletionManager + extends BasicLSMManager { + + // use wal manager object to write wal file on deletion + private WALManager walManager; + + public DeletionManager(WALManager walManager) { + this.walManager = walManager; + } + + /** + * write wal file on deletion + * + * @param root root memory node + * @param context request context + */ + @Override + public void preProcess(T root, R deletionRequest, DeleteRequestContext context) { + walManager.write(deletionRequest); + } + + @Override + public void postProcess(T root, R request, DeleteRequestContext context) {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/ILSMManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/ILSMManager.java new file mode 100644 index 000000000000..043bec8ddd4c --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/ILSMManager.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.manager; + +import org.apache.iotdb.lsm.context.requestcontext.RequestContext; +import org.apache.iotdb.lsm.levelProcess.LevelProcessorChain; +import org.apache.iotdb.lsm.request.IRequest; + +// Represents the manager of the lsm framework, used to handle root memory node +public interface ILSMManager { + + /** + * preprocessing of the root memory node + * + * @param root root memory node + * @param context request context + */ + void preProcess(T root, R request, C context); + + /** + * postprocessing of the root memory node + * + * @param root root memory node + * @param context request context + */ + void postProcess(T root, R request, C context); + + /** + * use this method to process root memory node + * + * @param memNode memory node + * @param context request context + */ + void process(T memNode, R request, C context); + + /** + * set level processors chain + * + * @param levelProcessorsChain level processors chain + */ + void setLevelProcessorsChain(LevelProcessorChain levelProcessorsChain); +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java new file mode 100644 index 000000000000..94e71d809bf7 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/InsertionManager.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.manager; + +import org.apache.iotdb.lsm.context.requestcontext.InsertRequestContext; +import org.apache.iotdb.lsm.request.IInsertionRequest; + +/** manage insertion to root memory node */ +public class InsertionManager + extends BasicLSMManager { + + // use wal manager object to write wal file on insertion + private WALManager walManager; + + public InsertionManager(WALManager walManager) { + this.walManager = walManager; + } + + /** + * write wal file on insertion + * + * @param root root memory node + * @param context insert request context + */ + @Override + public void preProcess(T root, R insertionRequest, InsertRequestContext context) { + walManager.write(insertionRequest); + } + + @Override + public void postProcess(T root, R request, InsertRequestContext context) {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java new file mode 100644 index 000000000000..a03d3df750b9 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/QueryManager.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.manager; + +import org.apache.iotdb.lsm.context.requestcontext.QueryRequestContext; +import org.apache.iotdb.lsm.request.IQueryRequest; + +/** manage query to root memory node */ +public class QueryManager + extends BasicLSMManager { + @Override + public void preProcess(T root, R request, QueryRequestContext context) {} + + @Override + public void postProcess(T root, R request, QueryRequestContext context) {} +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java new file mode 100644 index 000000000000..808f9b12c351 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/RecoverManager.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.manager; + +import org.apache.iotdb.lsm.engine.IRecoverable; +import org.apache.iotdb.lsm.request.IRequest; + +/** for memory structure recovery */ +public class RecoverManager { + + private WALManager walManager; + + public RecoverManager(WALManager walManager) { + this.walManager = walManager; + walManager.setRecover(true); + } + + /** + * recover + * + * @param t extends IRecoverable + */ + public void recover(T t) { + while (true) { + IRequest request = walManager.read(); + if (request == null) { + walManager.setRecover(false); + return; + } + t.recover(request); + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java new file mode 100644 index 000000000000..7ee7134e66a4 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/manager/WALManager.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.manager; + +import org.apache.iotdb.lsm.request.IRequest; +import org.apache.iotdb.lsm.wal.IWALRecord; +import org.apache.iotdb.lsm.wal.WALReader; +import org.apache.iotdb.lsm.wal.WALWriter; + +import java.io.File; +import java.io.IOException; + +/** Manage wal entry writes and reads */ +public abstract class WALManager { + + private final String walDirPath; + + private File walFile; + + // directly use the wal writer that comes with the lsm framework + private WALWriter walWriter; + + // directly use the wal reader that comes with the lsm framework + private WALReader walReader; + + private boolean recover; + + public WALManager(String schemaDirPath) { + this.walDirPath = schemaDirPath; + } + + public WALManager( + String walDirPath, + String walFileName, + int walBufferSize, + IWALRecord walRecord, + boolean forceEachWrite) + throws IOException { + this.walDirPath = walDirPath; + initFile(walDirPath, walFileName); + walWriter = new WALWriter(walFile, walBufferSize, forceEachWrite); + walReader = new WALReader(walFile, walRecord); + recover = false; + } + + private void initFile(String walDirPath, String walFileName) throws IOException { + File schemaDir = new File(walDirPath); + schemaDir.mkdirs(); + walFile = new File(this.walDirPath, walFileName); + if (!walFile.exists()) { + walFile.createNewFile(); + } + } + + /** + * handle wal log writes for each request + * + * @param request request context + * @throws IOException + */ + public abstract void write(IRequest request); + + /** + * for recover, read a wal record and generate it as a request + * + * @return request + */ + public abstract IRequest read(); + + public void close() throws IOException { + walWriter.close(); + walReader.close(); + } + + public String getSchemaDirPath() { + return walDirPath; + } + + public File getWalFile() { + return walFile; + } + + public void setWalFile(File walFile) { + this.walFile = walFile; + } + + public WALWriter getWalWriter() { + return walWriter; + } + + public void setWalWriter(WALWriter walWriter) { + this.walWriter = walWriter; + } + + public WALReader getWalReader() { + return walReader; + } + + public void setWalReader(WALReader walReader) { + this.walReader = walReader; + } + + public boolean isRecover() { + return recover; + } + + public void setRecover(boolean recover) { + this.recover = recover; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionRequest.java new file mode 100644 index 000000000000..e457fdf5e02f --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IDeletionRequest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.request; + +/** Represents a deletion request that can be processed by the lsm framework */ +public interface IDeletionRequest extends IRequest { + + RequestType requestType = RequestType.DELETE; + + @Override + default RequestType getRequestType() { + return requestType; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionRequest.java new file mode 100644 index 000000000000..b6c7e4ff983b --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IInsertionRequest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.request; + +/** Represents a insertion request that can be processed by the lsm framework */ +public interface IInsertionRequest extends IRequest { + + RequestType requestType = RequestType.INSERT; + + @Override + default RequestType getRequestType() { + return requestType; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryRequest.java new file mode 100644 index 000000000000..92ee326a353e --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IQueryRequest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.request; + +/** Represents a query request that can be processed by the lsm framework */ +public interface IQueryRequest extends IRequest { + + RequestType requestType = RequestType.QUERY; + + @Override + default Object getValue() { + return null; + } + + @Override + default RequestType getRequestType() { + return requestType; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IRequest.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IRequest.java new file mode 100644 index 000000000000..a2a810067c3d --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/IRequest.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.request; + +import org.apache.iotdb.lsm.context.requestcontext.RequestContext; + +import java.util.List; + +/** + * Represents a request that can be processed by the lsm framework + * + * @param The type of each layer key + * @param The type of value + */ +public interface IRequest { + + /** + * Get the key of a layer + * + * @param context request context + * @return the key of the layer + */ + K getKey(RequestContext context); + + /** + * get all keys + * + * @return all keys + */ + List getKeys(); + + /** + * get the value + * + * @return value of the request + */ + V getValue(); + + /** + * get request type + * + * @return request type + */ + RequestType getRequestType(); +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/RequestType.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/RequestType.java new file mode 100644 index 000000000000..59979044ec8e --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/request/RequestType.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.request; + +/** Indicates the request type of request */ +public enum RequestType { + NONE, + INSERT, + QUERY, + DELETE +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/response/IResponse.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/response/IResponse.java new file mode 100644 index 000000000000..420685577edb --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/response/IResponse.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.response; + +import java.util.List; + +/** + * Indicates the response after the lsm framework processes the request, encapsulating the response + * value and exception information. + * + * @param type of the response result + */ +public interface IResponse { + + T getValue(); + + void setValue(T value); + + List getExceptions(); + + void setExceptions(List exceptions); + + /** + * If an exception needs to be thrown during the processing of the request, this method can be + * used to accept the exception + * + * @param e Exception + */ + void addException(Exception e); +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java new file mode 100644 index 000000000000..b60f58371e5d --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/BFSAccessStrategy.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.requestcontext.RequestContext; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcessor; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; + +/** breadth-first access strategy implementation class */ +public class BFSAccessStrategy implements IAccessStrategy { + + // same level memory nodes, used to implement BFSAccessStrategy + Queue sameLevelMemNodes; + + /** + * breadth-first access strategy implementation + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ + @Override + public void execute( + BasicLevelProcessor levelProcess, I memNode, R request, C context) { + List children = new ArrayList<>(); + int currentLevel = context.getLevel(); + if (sameLevelMemNodes == null) { + sameLevelMemNodes = new LinkedList<>(); + // process the current memory node + levelProcess.handle(memNode, request, context); + // get all memory nodes to be processed in the next layer + children = levelProcess.getChildren(memNode, request, context); + } else { + while (!sameLevelMemNodes.isEmpty()) { + I node = (I) sameLevelMemNodes.poll(); + levelProcess.handle(node, request, context); + children.addAll(levelProcess.getChildren(node, request, context)); + } + } + sameLevelMemNodes.addAll(children); + context.setLevel(currentLevel + 1); + if (levelProcess.hasNext() && !sameLevelMemNodes.isEmpty()) { + levelProcess.getNext().process(null, request, context); + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/IAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/IAccessStrategy.java new file mode 100644 index 000000000000..2e95e5cb4535 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/IAccessStrategy.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.requestcontext.RequestContext; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcessor; + +/** access strategy for memory nodes */ +public interface IAccessStrategy { + + /** + * implementation of access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ + void execute( + BasicLevelProcessor levelProcess, I memNode, R request, C context); +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java new file mode 100644 index 000000000000..9095886f589f --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PostOrderAccessStrategy.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.requestcontext.RequestContext; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcessor; + +import java.util.List; + +/** post-order traversal access strategy implementation class */ +public class PostOrderAccessStrategy implements IAccessStrategy { + + /** + * post-order traversal access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ + @Override + public void execute( + BasicLevelProcessor levelProcess, I memNode, R request, C context) { + int currentLevel = context.getLevel(); + IAccessStrategy accessStrategy = context.getAccessStrategy(); + // get all memory nodes to be processed in the next layer + List children = levelProcess.getChildren(memNode, request, context); + if (levelProcess.hasNext()) { + context.setLevel(currentLevel + 1); + for (O child : children) { + // process next level memory node + levelProcess.getNext().process(child, request, context); + } + } + + context.setLevel(currentLevel); + context.setAccessStrategy(accessStrategy); + // process the current memory node + levelProcess.handle(memNode, request, context); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java new file mode 100644 index 000000000000..5f90f5ddf827 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/PreOrderAccessStrategy.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.requestcontext.RequestContext; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcessor; + +import java.util.List; + +/** pre-order traversal access strategy implementation class */ +public class PreOrderAccessStrategy implements IAccessStrategy { + + /** + * pre-order traversal access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ + @Override + public void execute( + BasicLevelProcessor levelProcess, I memNode, R request, C context) { + int currentLevel = context.getLevel(); + // process the current memory node + levelProcess.handle(memNode, request, context); + // get all memory nodes to be processed in the next layer + List children = levelProcess.getChildren(memNode, request, context); + + if (levelProcess.hasNext()) { + context.setLevel(currentLevel + 1); + for (O child : children) { + // process next level memory node + levelProcess.getNext().process(child, request, context); + } + } + context.setLevel(currentLevel); + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java new file mode 100644 index 000000000000..2daf9ecbeff0 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/strategy/RBFSAccessStrategy.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.strategy; + +import org.apache.iotdb.lsm.context.requestcontext.RequestContext; +import org.apache.iotdb.lsm.levelProcess.BasicLevelProcessor; + +import java.util.List; + +/** reverse breadth first traversal access strategy implementation class */ +public class RBFSAccessStrategy implements IAccessStrategy { + + /** + * reverse breadth first traversal access strategy + * + * @param levelProcess current level process + * @param memNode memory node + * @param context request context + */ + @Override + public void execute( + BasicLevelProcessor levelProcess, I memNode, R request, C context) { + int currentLevel = context.getLevel(); + + // if the upper bound has not been set and there is no next-level processing method, set the + // upper bound to the current level + if (Integer.MAX_VALUE == context.getLevelUpperBound() && !levelProcess.hasNext()) { + context.setLevelUpperBound(context.getLevel()); + } + + // if the current memory node is the root + if (currentLevel == 0) { + // if all the next level nodes of the root node have not been processed + while (context.getLevelUpperBound() != currentLevel) { + // process all pending next-level nodes + List children = levelProcess.getChildren(memNode, request, context); + for (O child : children) { + context.setLevel(currentLevel + 1); + // use the processing method of the next layer to process the next layer of nodes + levelProcess.getNext().process(child, request, context); + context.setLevel(currentLevel); + } + + // after each layer is processed, the upper bound is reduced by one + context.setLevelUpperBound(context.getLevelUpperBound() - 1); + } + + // process the current memory node + levelProcess.handle(memNode, request, context); + return; + } + + if (currentLevel > context.getLevelUpperBound()) return; + + // only process memory nodes with equal level and upper bound + if (currentLevel == context.getLevelUpperBound()) { + levelProcess.handle(memNode, request, context); + return; + } + + // process all pending next-level nodes + List children = levelProcess.getChildren(memNode, request, context); + for (O child : children) { + context.setLevel(currentLevel + 1); + levelProcess.getNext().process(child, request, context); + context.setLevel(currentLevel); + } + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java new file mode 100644 index 000000000000..5309737fb695 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALReader.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import java.io.FileNotFoundException; +import java.io.IOException; + +/** get records in wal file */ +public interface IWALReader { + + /** + * close resource + * + * @throws IOException + */ + void close() throws IOException; + + /** + * determine if there is a next record + * + * @return returns true if there is, else returns false + * @throws FileNotFoundException + */ + boolean hasNext() throws FileNotFoundException; + + /** + * return the next record + * + * @throws FileNotFoundException + */ + IWALRecord next() throws FileNotFoundException; +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALRecord.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALRecord.java new file mode 100644 index 000000000000..18e48fb32516 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALRecord.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import java.io.DataInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +/** represents a wal record, which can be extended to implement more complex wal records */ +public interface IWALRecord extends Cloneable { + + /** + * serialize the wal record + * + * @param buffer byte buffer + */ + void serialize(ByteBuffer buffer); + + /** + * deserialize via input stream + * + * @param stream data input stream + * @throws IOException + */ + void deserialize(DataInputStream stream) throws IOException; + + // generate wal record using prototyping pattern + IWALRecord clone(); + + List getKeys(); + + V getValue(); +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java new file mode 100644 index 000000000000..4a41b5aa212b --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/IWALWriter.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import java.io.IOException; + +/** write records to wal file */ +public interface IWALWriter { + + /** + * write walRecord to wal file + * + * @param walRecord record to be written + * @throws IOException + */ + void write(IWALRecord walRecord) throws IOException; + + /** + * force brush + * + * @throws IOException + */ + void force() throws IOException; + + /** + * close resource + * + * @throws IOException + */ + void close() throws IOException; +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java new file mode 100644 index 000000000000..63469afdda2d --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALReader.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.NoSuchElementException; + +/** get records in wal file */ +public class WALReader implements IWALReader { + private static final Logger logger = LoggerFactory.getLogger(WALReader.class); + // wal file + private final File logFile; + // wal record prototype, clone on read + private final IWALRecord prototype; + private DataInputStream logStream; + // next wal record + private IWALRecord nextRecord; + private boolean fileCorrupted = false; + + public WALReader(File logFile, IWALRecord prototype) throws IOException { + this.logFile = logFile; + this.logStream = + new DataInputStream(new BufferedInputStream(Files.newInputStream(logFile.toPath()))); + this.prototype = prototype; + } + + @Override + public void close() throws IOException { + logStream.close(); + logStream = null; + } + + @Override + public boolean hasNext() { + if (nextRecord != null) { + return true; + } + try { + if (fileCorrupted) { + return false; + } + int logSize = logStream.readInt(); + if (logSize <= 0) { + return false; + } + // first clone the object through the prototype + nextRecord = prototype.clone(); + // then perform deserialization and assign a value to the new object + nextRecord.deserialize(logStream); + } catch (EOFException e) { + logger.info(e.getMessage()); + return false; + } catch (IOException e) { + logger.warn(e.getMessage()); + fileCorrupted = true; + return false; + } + return true; + } + + @Override + public IWALRecord next() { + if (nextRecord == null) { + throw new NoSuchElementException(); + } + IWALRecord walRecord = nextRecord; + nextRecord = null; + return walRecord; + } + + @Override + public String toString() { + return "WALReader{" + "logFile=" + logFile + '}'; + } +} diff --git a/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java new file mode 100644 index 000000000000..9b3924c01586 --- /dev/null +++ b/schema-engine-tag/src/main/java/org/apache/iotdb/lsm/wal/WALWriter.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.lsm.wal; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.FileChannel; + +/** write records to wal file */ +public class WALWriter implements IWALWriter { + private static final Logger logger = LoggerFactory.getLogger(WALWriter.class); + // wal file + private File logFile; + private FileOutputStream fileOutputStream; + private FileChannel channel; + // 4-bit buffer + private final ByteBuffer lengthBuffer; + // save wal record serialized byte data + private final ByteBuffer walBuffer; + private final boolean forceEachWrite; + + public WALWriter(File logFile, int walBufferSize, boolean forceEachWrite) + throws FileNotFoundException { + this.logFile = logFile; + this.forceEachWrite = forceEachWrite; + fileOutputStream = new FileOutputStream(logFile, true); + channel = fileOutputStream.getChannel(); + lengthBuffer = ByteBuffer.allocate(4); + walBuffer = ByteBuffer.allocate(walBufferSize); + } + + /** + * write walRecord to wal file + * + * @param walRecord record to be written + * @throws IOException + */ + @Override + public void write(IWALRecord walRecord) throws IOException { + if (channel == null) { + fileOutputStream = new FileOutputStream(logFile, true); + channel = fileOutputStream.getChannel(); + } + walBuffer.clear(); + walRecord.serialize(walBuffer); + walBuffer.flip(); + int logSize = walBuffer.limit(); + lengthBuffer.clear(); + lengthBuffer.putInt(logSize); + lengthBuffer.flip(); + + try { + channel.write(lengthBuffer); + channel.write(walBuffer); + + if (this.forceEachWrite) { + channel.force(true); + } + } catch (ClosedChannelException ignored) { + logger.warn("someone interrupt current thread, so no need to do write for io safety"); + } + } + + @Override + public void force() throws IOException { + if (channel != null && channel.isOpen()) { + channel.force(true); + } + } + + @Override + public void close() throws IOException { + if (channel != null) { + if (channel.isOpen()) { + channel.force(true); + } + fileOutputStream.close(); + fileOutputStream = null; + channel.close(); + channel = null; + } + } + + @Override + public String toString() { + return "WALLogWriter{" + "logFile=" + logFile + '}'; + } +} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegionTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegionTest.java new file mode 100644 index 000000000000..5153de805afe --- /dev/null +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/TagSchemaRegionTest.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion; + +import org.apache.iotdb.commons.consensus.SchemaRegionId; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class TagSchemaRegionTest { + + private CompressionType compressionType; + + private String storageGroupDirPath; + + private String schemaRegionDirPath; + + private String storageGroupFullPath = "root/testTagSchemaRegion"; + + private String storageGroup = "root.testTagSchemaRegion"; + + private boolean isEnableIDTable = false; + + private String originalDeviceIDTransformationMethod = null; + + private boolean isEnableIDTableLogFile = false; + + private String schemaDir; + + private TagSchemaRegion tagSchemaRegion; + + @Before + public void before() { + compressionType = TSFileDescriptor.getInstance().getConfig().getCompressor(); + schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir(); + isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable(); + originalDeviceIDTransformationMethod = + IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); + isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile(); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true); + storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath; + schemaRegionDirPath = storageGroupDirPath + File.separator + 0; + } + + @After + public void clean() { + tagSchemaRegion.clear(); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); + IoTDBDescriptor.getInstance() + .getConfig() + .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile); + FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir)); + } + + @Test + public void testCreateTimeseries() { + try { + tagSchemaRegion = + new TagSchemaRegion(new PartialPath(storageGroup), new SchemaRegionId(0), null, null); + createTimeseries(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testCreateAlignedTimeSeries() { + try { + tagSchemaRegion = + new TagSchemaRegion(new PartialPath(storageGroup), new SchemaRegionId(0), null, null); + createAlignedTimeseries(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testRecover() { + try { + tagSchemaRegion = + new TagSchemaRegion(new PartialPath(storageGroup), new SchemaRegionId(0), null, null); + createAlignedTimeseries(); + createTimeseries(); + + tagSchemaRegion.clear(); + tagSchemaRegion = + new TagSchemaRegion(new PartialPath(storageGroup), new SchemaRegionId(0), null, null); + + getMeasurementPathsTest(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testGetMeasurementPaths() { + try { + tagSchemaRegion = + new TagSchemaRegion(new PartialPath(storageGroup), new SchemaRegionId(0), null, null); + createAlignedTimeseries(); + createTimeseries(); + getMeasurementPathsTest(); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private void getMeasurementPathsTest() throws Exception { + PartialPath pathPattern = new PartialPath(storageGroup + ".tag1.a.tag2.b.s0"); + List measurementPaths = + tagSchemaRegion.getMeasurementPaths(pathPattern, false, false); + assertEquals(measurementPaths.size(), 1); + assertEquals(measurementPaths.get(0).getFullPath(), storageGroup + ".tag1.a.tag2.b.s0"); + + pathPattern = new PartialPath(storageGroup + ".tag1.a.**.s"); + measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false, false); + assertEquals(measurementPaths.size(), 5); + + pathPattern = new PartialPath(storageGroup + ".tag2.b.**.s"); + measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false, false); + assertEquals(measurementPaths.size(), 1); + + pathPattern = new PartialPath(storageGroup + ".tag3.b.**.s"); + measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false, false); + assertEquals(measurementPaths.size(), 3); + + pathPattern = new PartialPath(storageGroup + ".tag2.y.tag1.x.**.s"); + measurementPaths = tagSchemaRegion.getMeasurementPaths(pathPattern, false, false); + assertEquals(measurementPaths.size(), 3); + } + + private void createTimeseries() throws Exception { + CreateTimeSeriesPlan createTimeSeriesPlan = + new CreateTimeSeriesPlan( + new PartialPath(storageGroup + ".tag1.a.tag2.b.s0"), + TSDataType.valueOf("INT32"), + TSEncoding.valueOf("RLE"), + compressionType, + Collections.emptyMap(), + null, + null, + null); + tagSchemaRegion.createTimeseries(createTimeSeriesPlan, 0); + createTimeSeriesPlan = + new CreateTimeSeriesPlan( + new PartialPath(storageGroup + ".tag1.a.tag2.c.s0"), + TSDataType.valueOf("INT32"), + TSEncoding.valueOf("RLE"), + compressionType, + Collections.emptyMap(), + null, + null, + null); + tagSchemaRegion.createTimeseries(createTimeSeriesPlan, 0); + } + + private void createAlignedTimeseries() throws Exception { + CreateAlignedTimeSeriesPlan plan = + new CreateAlignedTimeSeriesPlan( + new PartialPath(storageGroup + ".tag1.a.tag3.b"), + Arrays.asList("s1", "s2", "s3"), + Arrays.asList( + TSDataType.valueOf("FLOAT"), + TSDataType.valueOf("INT64"), + TSDataType.valueOf("INT32")), + Arrays.asList( + TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE")), + Arrays.asList(compressionType, compressionType, compressionType), + null, + null, + null); + tagSchemaRegion.createAlignedTimeSeries(plan); + plan = + new CreateAlignedTimeSeriesPlan( + new PartialPath(storageGroup + ".tag1.x.tag2.y"), + Arrays.asList("s1", "s2", "s3"), + Arrays.asList( + TSDataType.valueOf("FLOAT"), + TSDataType.valueOf("INT64"), + TSDataType.valueOf("INT32")), + Arrays.asList( + TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE")), + Arrays.asList(compressionType, compressionType, compressionType), + null, + null, + null); + tagSchemaRegion.createAlignedTimeSeries(plan); + } +} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImplTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImplTest.java new file mode 100644 index 000000000000..2e9b4ccde805 --- /dev/null +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/idtable/IDTableWithDeviceIDListImplTest.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.idtable; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.StorageEngineException; +import org.apache.iotdb.db.qp.physical.sys.CreateAlignedTimeSeriesPlan; +import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan; +import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class IDTableWithDeviceIDListImplTest { + + private CompressionType compressionType; + + private String storageGroupDirPath; + + private String schemaRegionDirPath; + + private String storageGroupFullPath = "root/testIDTableWithDeviceIDListImpl"; + + private String storageGroup = "root.testIDTableWithDeviceIDListImpl"; + + private boolean isEnableIDTable = false; + + private String originalDeviceIDTransformationMethod = null; + + private boolean isEnableIDTableLogFile = false; + + private String schemaDir; + + private IDTableWithDeviceIDListImpl idTableWithDeviceIDList; + + @Before + public void before() { + compressionType = TSFileDescriptor.getInstance().getConfig().getCompressor(); + schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir(); + isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable(); + originalDeviceIDTransformationMethod = + IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod(); + isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile(); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true); + IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256"); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true); + storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath; + schemaRegionDirPath = storageGroupDirPath + File.separator + 0; + idTableWithDeviceIDList = new IDTableWithDeviceIDListImpl(new File(schemaRegionDirPath)); + } + + @After + public void clean() throws IOException, StorageEngineException { + idTableWithDeviceIDList.clear(); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable); + IoTDBDescriptor.getInstance() + .getConfig() + .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod); + IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile); + FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir)); + } + + @Test + public void testCreateAlignedTimeseries() { + try { + createAlignedTimeseries(); + assertEquals(idTableWithDeviceIDList.size(), 2); + assertEquals( + idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d1.aligned_device").getDeviceID(), + idTableWithDeviceIDList.get(0)); + assertEquals( + idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d2.aligned_device").getDeviceID(), + idTableWithDeviceIDList.get(1)); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testCreateTimeseries() { + try { + createTimeseries(); + assertEquals(idTableWithDeviceIDList.size(), 2); + assertEquals( + idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d1").getDeviceID(), + idTableWithDeviceIDList.get(0)); + assertEquals( + idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d2").getDeviceID(), + idTableWithDeviceIDList.get(1)); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testRecover() { + try { + createTimeseries(); + createAlignedTimeseries(); + + idTableWithDeviceIDList.clear(); + idTableWithDeviceIDList = new IDTableWithDeviceIDListImpl(new File(schemaRegionDirPath)); + + assertEquals(idTableWithDeviceIDList.size(), 4); + assertEquals( + idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d1").getDeviceID(), + idTableWithDeviceIDList.get(0)); + assertEquals( + idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d2").getDeviceID(), + idTableWithDeviceIDList.get(1)); + assertEquals( + idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d1.aligned_device").getDeviceID(), + idTableWithDeviceIDList.get(2)); + assertEquals( + idTableWithDeviceIDList.getDeviceEntry(storageGroup + ".d2.aligned_device").getDeviceID(), + idTableWithDeviceIDList.get(3)); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private void createAlignedTimeseries() throws Exception { + CreateAlignedTimeSeriesPlan plan = + new CreateAlignedTimeSeriesPlan( + new PartialPath(storageGroup + ".d1.aligned_device"), + Arrays.asList("s1", "s2", "s3"), + Arrays.asList( + TSDataType.valueOf("FLOAT"), + TSDataType.valueOf("INT64"), + TSDataType.valueOf("INT32")), + Arrays.asList( + TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE")), + Arrays.asList(compressionType, compressionType, compressionType), + null, + null, + null); + idTableWithDeviceIDList.createAlignedTimeseries(plan); + plan = + new CreateAlignedTimeSeriesPlan( + new PartialPath(storageGroup + ".d2.aligned_device"), + Arrays.asList("s1", "s2", "s3"), + Arrays.asList( + TSDataType.valueOf("FLOAT"), + TSDataType.valueOf("INT64"), + TSDataType.valueOf("INT32")), + Arrays.asList( + TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE"), TSEncoding.valueOf("RLE")), + Arrays.asList(compressionType, compressionType, compressionType), + null, + null, + null); + idTableWithDeviceIDList.createAlignedTimeseries(plan); + } + + private void createTimeseries() throws Exception { + CreateTimeSeriesPlan createTimeSeriesPlan = + new CreateTimeSeriesPlan( + new PartialPath(storageGroup + ".d1.s0"), + TSDataType.valueOf("INT32"), + TSEncoding.valueOf("RLE"), + compressionType, + Collections.emptyMap(), + null, + null, + null); + idTableWithDeviceIDList.createTimeseries(createTimeSeriesPlan); + createTimeSeriesPlan = + new CreateTimeSeriesPlan( + new PartialPath(storageGroup + ".d2.s0"), + TSDataType.valueOf("INT32"), + TSEncoding.valueOf("RLE"), + compressionType, + Collections.emptyMap(), + null, + null, + null); + idTableWithDeviceIDList.createTimeseries(createTimeSeriesPlan); + } +} diff --git a/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagTagInvertedIndexTest.java b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagTagInvertedIndexTest.java new file mode 100644 index 000000000000..b46d863b171c --- /dev/null +++ b/schema-engine-tag/src/test/java/org/apache/iotdb/db/metadata/tagSchemaRegion/tagIndex/TagTagInvertedIndexTest.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.metadata.tagSchemaRegion.tagIndex; + +import org.apache.iotdb.commons.utils.FileUtils; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.metadata.tagSchemaRegion.config.TagSchemaDescriptor; +import org.apache.iotdb.tsfile.utils.Pair; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +public class TagTagInvertedIndexTest { + private String[][] record = + new String[][] { + {"tag1=q", "tag2=a", "1"}, + {"tag1=q", "tag2=s", "2"}, + {"tag1=q", "tag2=a", "tag3=z", "3"}, + {"tag1=q", "tag3=v", "4"}, + {"tag1=q", "tag2=s", "5"}, + {"tag1=w", "tag2=d", "6"}, + {"tag1=q", "tag2=d", "tag3=e", "7"}, + {"tag1=t", "tag2=g", "8"}, + {"tag1=r", "tag2=d", "9"}, + {"tag1=t", "tag2=f", "10"}, + {"tag1=t", "tag2=h", "11"}, + {"tag1=q", "tag2=a", "tag3=l", "12"}, + {"tag1=y", "tag2=j", "13"}, + {"tag1=u", "tag2=k", "14"}, + {"tag1=q", "tag2=a", "tag3=x", "15"}, + {"tag1=q", "tag2=a", "tag4=z", "16"}, + {"tag1=y", "tag2=a", "tag4=z", "17"}, + {"tag1=q", "tag2=b", "tag3=x", "18"}, + }; + + private int numOfDeviceIdsInMemTable; + + private TagInvertedIndex tagInvertedIndex; + + private String storageGroupDirPath; + + private String schemaRegionDirPath; + + private String storageGroupFullPath = "root/testTagIndex"; + + private String schemaDir; + + @Before + public void setUp() throws Exception { + numOfDeviceIdsInMemTable = + TagSchemaDescriptor.getInstance().getTagSchemaConfig().getNumOfDeviceIdsInMemTable(); + TagSchemaDescriptor.getInstance().getTagSchemaConfig().setNumOfDeviceIdsInMemTable(3); + schemaDir = IoTDBDescriptor.getInstance().getConfig().getSchemaDir(); + storageGroupDirPath = schemaDir + File.separator + storageGroupFullPath; + schemaRegionDirPath = storageGroupDirPath + File.separator + 0; + tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath); + } + + @After + public void tearDown() throws Exception { + TagSchemaDescriptor.getInstance() + .getTagSchemaConfig() + .setNumOfDeviceIdsInMemTable(numOfDeviceIdsInMemTable); + tagInvertedIndex.clear(); + tagInvertedIndex = null; + FileUtils.deleteDirectoryAndEmptyParent(new File(schemaDir)); + } + + public void addTags() { + List, Integer>> records = generateTags(); + for (Pair, Integer> pair : records) { + tagInvertedIndex.addTags(pair.left, pair.right); + } + } + + public void removeTags() { + Pair, Integer> tags = generateTag(record[0]); + tagInvertedIndex.removeTags(tags.left, tags.right); + tags = generateTag(record[1]); + tagInvertedIndex.removeTags(tags.left, tags.right); + tags = generateTag(record[3]); + tagInvertedIndex.removeTags(tags.left, tags.right); + tags = generateTag(record[11]); + tagInvertedIndex.removeTags(tags.left, tags.right); + } + + @Test + public void getMatchedIDs() { + addTags(); + Map tags1 = new HashMap<>(); + tags1.put("tag1", "q"); + + Map tags2 = new HashMap<>(); + tags2.put("tag1", "q"); + tags2.put("tag2", "a"); + + List ids = tagInvertedIndex.getMatchedIDs(tags1); + List verify = Arrays.asList(1, 2, 3, 4, 5, 7, 12, 15, 16, 18); + assertEquals(verify, ids); + + ids = tagInvertedIndex.getMatchedIDs(tags2); + verify = Arrays.asList(1, 3, 12, 15, 16); + assertEquals(verify, ids); + + removeTags(); + + ids = tagInvertedIndex.getMatchedIDs(tags1); + verify = Arrays.asList(3, 5, 7, 15, 16, 18); + assertEquals(verify, ids); + + ids = tagInvertedIndex.getMatchedIDs(tags2); + verify = Arrays.asList(3, 15, 16); + assertEquals(verify, ids); + } + + @Test + public void testRecover() throws IOException { + Map tags1 = new HashMap<>(); + tags1.put("tag1", "q"); + + Map tags2 = new HashMap<>(); + tags2.put("tag1", "q"); + tags2.put("tag2", "a"); + addTags(); + removeTags(); + + tagInvertedIndex.clear(); + tagInvertedIndex = new TagInvertedIndex(schemaRegionDirPath); + + List ids = tagInvertedIndex.getMatchedIDs(tags1); + List verify = Arrays.asList(3, 5, 7, 15, 16, 18); + assertEquals(verify, ids); + + ids = tagInvertedIndex.getMatchedIDs(tags2); + verify = Arrays.asList(3, 15, 16); + assertEquals(verify, ids); + } + + private List, Integer>> generateTags() { + List, Integer>> pairs = new ArrayList<>(); + for (String[] strings : record) { + pairs.add(generateTag(strings)); + } + return pairs; + } + + private Pair, Integer> generateTag(String[] strings) { + Map tags = new HashMap<>(); + int i = 0; + for (; i < strings.length - 1; i++) { + String[] str = strings[i].split("="); + tags.put(str[0], str[1]); + } + Pair, Integer> pair = new Pair<>(tags, Integer.valueOf(strings[i])); + return pair; + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java index e0e21ddbfaa6..66cb8eb3c1fd 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DiskSchemaEntry.java @@ -74,6 +74,15 @@ public DiskSchemaEntry( this.isAligned = isAligned; } + /** + * get device path + * + * @return device path + */ + public String getDevicePath() { + return seriesKey.substring(0, seriesKey.length() - measurementName.length() - 1); + } + public int serialize(OutputStream outputStream) throws IOException { int byteLen = 0; byteLen += ReadWriteIOUtils.write(deviceID, outputStream); diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java index ac7c1ce9842e..a19f3682fa68 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/dto/IoTDBPoint.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.query.QueryProcessException; -import org.apache.iotdb.db.protocol.influxdb.meta.AbstractInfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.IInfluxDBMetaManager; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; import org.apache.iotdb.db.utils.DataTypeUtils; import org.apache.iotdb.db.utils.ParameterUtils; @@ -37,6 +37,10 @@ import java.util.Map; import java.util.concurrent.TimeUnit; +/** + * Represent an IoTDB point, including the device path to be written, the measurement point and the + * corresponding value + */ public class IoTDBPoint { private final String deviceId; @@ -59,7 +63,7 @@ public IoTDBPoint( } public IoTDBPoint( - String database, Point point, AbstractInfluxDBMetaManager metaManager, long sessionID) { + String database, Point point, IInfluxDBMetaManager influxDBMetaManager, long sessionID) { String measurement = null; Map tags = new HashMap<>(); Map fields = new HashMap<>(); @@ -105,7 +109,8 @@ public IoTDBPoint( } ParameterUtils.checkNonEmptyString(database, "database"); ParameterUtils.checkNonEmptyString(measurement, "measurement name"); - String path = metaManager.generatePath(database, measurement, tags, sessionID); + String path = + influxDBMetaManager.generatePath(database, measurement, tags, fields.keySet(), sessionID); List measurements = new ArrayList<>(); List types = new ArrayList<>(); List values = new ArrayList<>(); @@ -152,6 +157,13 @@ public InsertRowPlan convertToInsertRowPlan() false); } + /** + * Convert IoTDB point to InsertRecordReq + * + * @param sessionID session id + * @return InsertRecordReq + * @throws IoTDBConnectionException + */ public TSInsertRecordReq convertToTSInsertRecordReq(long sessionID) throws IoTDBConnectionException { TSInsertRecordReq tsInsertRecordReq = new TSInsertRecordReq(); diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java index 78899f7e34bb..10cdef4cd04d 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/AbstractQueryHandler.java @@ -29,7 +29,7 @@ import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; import org.apache.iotdb.db.protocol.influxdb.function.aggregator.InfluxAggregator; import org.apache.iotdb.db.protocol.influxdb.function.selector.InfluxSelector; -import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; import org.apache.iotdb.db.protocol.influxdb.operator.InfluxQueryOperator; import org.apache.iotdb.db.protocol.influxdb.operator.InfluxSelectComponent; import org.apache.iotdb.db.protocol.influxdb.util.FilterUtils; @@ -40,7 +40,6 @@ import org.apache.iotdb.db.qp.logical.Operator; import org.apache.iotdb.db.qp.logical.crud.BasicFunctionOperator; import org.apache.iotdb.db.qp.logical.crud.FilterOperator; -import org.apache.iotdb.db.service.basic.ServiceProvider; import org.apache.iotdb.protocol.influxdb.rpc.thrift.InfluxQueryResultRsp; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -55,32 +54,50 @@ import java.util.List; import java.util.Map; +/** Used to process influxdb query requests, this abstract class defines some template methods */ public abstract class AbstractQueryHandler { - abstract Map getFieldOrders( - String database, String measurement, ServiceProvider serviceProvider, long sessionId); - + /** + * If the function in the influxdb query request is also supported by IoTDB, use the IoTDB syntax + * to get the result + * + * @param database database of influxdb + * @param measurement measurement of influxdb + * @param function influxdb function + * @param sessionid session id + * @return influxdb function value + */ abstract InfluxFunctionValue updateByIoTDBFunc( - InfluxFunction function, ServiceProvider serviceProvider, String path, long sessionid); + String database, String measurement, InfluxFunction function, long sessionid); + /** + * The method that needs to be implemented is to query the result according to the query SQL + * supported by IoTDB + * + * @param querySql query sql + * @param database database of influxdb + * @param measurement measurement of influxdb + * @param tagOrders tag orders + * @param fieldOrders field orders + * @param sessionId session id + * @return query result + * @throws AuthException + */ abstract QueryResult queryByConditions( String querySql, String database, String measurement, - ServiceProvider serviceProvider, + Map tagOrders, Map fieldOrders, long sessionId) throws AuthException; public final InfluxQueryResultRsp queryInfluxDB( - String database, - InfluxQueryOperator queryOperator, - long sessionId, - ServiceProvider serviceProvider) { + String database, InfluxQueryOperator queryOperator, long sessionId) { String measurement = queryOperator.getFromComponent().getPrefixPaths().get(0).getFullPath(); // The list of fields under the current measurement and the order of the specified rules Map fieldOrders = - getFieldOrders(database, measurement, serviceProvider, sessionId); + InfluxDBMetaManagerFactory.getInstance().getFieldOrders(database, measurement, sessionId); QueryResult queryResult; InfluxQueryResultRsp tsQueryResultRsp = new InfluxQueryResultRsp(); try { @@ -96,7 +113,6 @@ public final InfluxQueryResultRsp queryInfluxDB( : null, database, measurement, - serviceProvider, fieldOrders, sessionId); // step2 : select filter @@ -106,11 +122,7 @@ public final InfluxQueryResultRsp queryInfluxDB( else { queryResult = queryFuncWithoutFilter( - queryOperator.getSelectComponent(), - database, - measurement, - serviceProvider, - sessionId); + queryOperator.getSelectComponent(), database, measurement, sessionId); } return tsQueryResultRsp .setResultJsonString(JacksonUtils.bean2Json(queryResult)) @@ -274,18 +286,14 @@ else if (selectComponent.isHasCommonQuery()) { * @param selectComponent select data to query * @return select query result */ - public final QueryResult queryFuncWithoutFilter( - InfluxSelectComponent selectComponent, - String database, - String measurement, - ServiceProvider serviceProvider, - long sessionid) { + public QueryResult queryFuncWithoutFilter( + InfluxSelectComponent selectComponent, String database, String measurement, long sessionid) { // columns List columns = new ArrayList<>(); columns.add(InfluxSQLConstant.RESERVED_TIME); List functions = new ArrayList<>(); - String path = "root." + database + "." + measurement; + for (ResultColumn resultColumn : selectComponent.getResultColumns()) { Expression expression = resultColumn.getExpression(); if (expression instanceof FunctionExpression) { @@ -300,7 +308,7 @@ public final QueryResult queryFuncWithoutFilter( List> values = new ArrayList<>(); for (InfluxFunction function : functions) { InfluxFunctionValue functionValue = - updateByIoTDBFunc(function, serviceProvider, path, sessionid); + updateByIoTDBFunc(database, measurement, function, sessionid); // InfluxFunctionValue functionValue = function.calculateByIoTDBFunc(); if (value.size() == 0) { value.add(functionValue.getTimestamp()); @@ -330,40 +338,33 @@ public QueryResult queryExpr( FilterOperator operator, String database, String measurement, - ServiceProvider serviceProvider, Map fieldOrders, Long sessionId) throws AuthException { if (operator == null) { List expressions = new ArrayList<>(); - return queryByConditions( - expressions, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(expressions, database, measurement, fieldOrders, sessionId); } else if (operator instanceof BasicFunctionOperator) { List iExpressions = new ArrayList<>(); iExpressions.add(getIExpressionForBasicFunctionOperator((BasicFunctionOperator) operator)); - return queryByConditions( - iExpressions, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(iExpressions, database, measurement, fieldOrders, sessionId); } else { FilterOperator leftOperator = operator.getChildren().get(0); FilterOperator rightOperator = operator.getChildren().get(1); if (operator.getFilterType() == FilterConstant.FilterType.KW_OR) { return QueryResultUtils.orQueryResultProcess( - queryExpr(leftOperator, database, measurement, serviceProvider, fieldOrders, sessionId), - queryExpr( - rightOperator, database, measurement, serviceProvider, fieldOrders, sessionId)); + queryExpr(leftOperator, database, measurement, fieldOrders, sessionId), + queryExpr(rightOperator, database, measurement, fieldOrders, sessionId)); } else if (operator.getFilterType() == FilterConstant.FilterType.KW_AND) { if (canMergeOperator(leftOperator) && canMergeOperator(rightOperator)) { List iExpressions1 = getIExpressionByFilterOperatorOperator(leftOperator); List iExpressions2 = getIExpressionByFilterOperatorOperator(rightOperator); iExpressions1.addAll(iExpressions2); - return queryByConditions( - iExpressions1, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(iExpressions1, database, measurement, fieldOrders, sessionId); } else { return QueryResultUtils.andQueryResultProcess( - queryExpr( - leftOperator, database, measurement, serviceProvider, fieldOrders, sessionId), - queryExpr( - rightOperator, database, measurement, serviceProvider, fieldOrders, sessionId)); + queryExpr(leftOperator, database, measurement, fieldOrders, sessionId), + queryExpr(rightOperator, database, measurement, fieldOrders, sessionId)); } } } @@ -376,11 +377,10 @@ public QueryResult queryExpr( * @param expressions list of conditions, including tag and field condition * @return returns the results of the influxdb query */ - private QueryResult queryByConditions( + public QueryResult queryByConditions( List expressions, String database, String measurement, - ServiceProvider serviceProvider, Map fieldOrders, Long sessionId) throws AuthException { @@ -390,7 +390,8 @@ private QueryResult queryByConditions( List fieldExpressions = new ArrayList<>(); // maximum number of tags in the current query criteria int currentQueryMaxTagNum = 0; - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionId); for (IExpression expression : expressions) { SingleSeriesExpression singleSeriesExpression = ((SingleSeriesExpression) expression); // the current condition is in tag @@ -445,8 +446,7 @@ private QueryResult queryByConditions( realQuerySql += " where " + realIotDBCondition; } realQuerySql += " align by device"; - return queryByConditions( - realQuerySql, database, measurement, serviceProvider, fieldOrders, sessionId); + return queryByConditions(realQuerySql, database, measurement, null, fieldOrders, sessionId); } /** diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java index ee8d0db5c06b..0d10431e991a 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/NewQueryHandler.java @@ -18,77 +18,41 @@ */ package org.apache.iotdb.db.protocol.influxdb.handler; -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; import org.apache.iotdb.db.protocol.influxdb.constant.InfluxSQLConstant; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunction; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; -import org.apache.iotdb.db.protocol.influxdb.meta.NewInfluxDBMetaManager; import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; -import org.apache.iotdb.db.service.basic.ServiceProvider; import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; -import org.apache.iotdb.rpc.TSStatusCode; -import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; -import org.influxdb.InfluxDBException; import org.influxdb.dto.QueryResult; -import java.util.HashMap; import java.util.List; import java.util.Map; +/** query handler for NewIoTDB */ public class NewQueryHandler extends AbstractQueryHandler { - public static TSExecuteStatementResp executeStatement(String sql, long sessionId) { - TSExecuteStatementReq tsExecuteStatementReq = new TSExecuteStatementReq(); - tsExecuteStatementReq.setStatement(sql); - tsExecuteStatementReq.setSessionId(sessionId); - tsExecuteStatementReq.setStatementId( - NewInfluxDBServiceImpl.getClientRPCService().requestStatementId(sessionId)); - tsExecuteStatementReq.setFetchSize(InfluxConstant.DEFAULT_FETCH_SIZE); - TSExecuteStatementResp executeStatementResp = - NewInfluxDBServiceImpl.getClientRPCService().executeStatement(tsExecuteStatementReq); - TSStatus tsStatus = executeStatementResp.getStatus(); - if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - throw new InfluxDBException(tsStatus.getMessage()); - } - return executeStatementResp; - } - - @Override - public Map getFieldOrders( - String database, String measurement, ServiceProvider serviceProvider, long sessionID) { - Map fieldOrders = new HashMap<>(); - String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; - TSExecuteStatementResp executeStatementResp = executeStatement(showTimeseriesSql, sessionID); - List paths = QueryResultUtils.getFullPaths(executeStatementResp); - Map tagOrders = NewInfluxDBMetaManager.getTagOrders(database, measurement); - int tagOrderNums = tagOrders.size(); - int fieldNums = 0; - for (String path : paths) { - String filed = StringUtils.getFieldByPath(path); - if (!fieldOrders.containsKey(filed)) { - // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, - // and finally all fields) - fieldOrders.put(filed, tagOrderNums + fieldNums + 1); - fieldNums++; - } - } - return fieldOrders; - } - - @Override - public InfluxFunctionValue updateByIoTDBFunc( - InfluxFunction function, ServiceProvider serviceProvider, String path, long sessionid) { + /** + * If the function in the influxdb query request is also supported by IoTDB, use the IoTDB syntax + * to get the result + * + * @param path storage group path + * @param function influxdb function + * @param sessionid session id + * @return influxdb function value + */ + public final InfluxFunctionValue updateByIoTDBFunc( + String path, InfluxFunction function, long sessionid) { switch (function.getFunctionName()) { case InfluxSQLConstant.COUNT: { String functionSql = StringUtils.generateFunctionSql( function.getFunctionName(), function.getParmaName(), path); - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -101,7 +65,7 @@ public InfluxFunctionValue updateByIoTDBFunc( String functionSqlCount = StringUtils.generateFunctionSql("count", function.getParmaName(), path); TSExecuteStatementResp tsExecuteStatementResp = - executeStatement(functionSqlCount, sessionid); + NewInfluxDBServiceImpl.executeStatement(functionSqlCount, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -109,7 +73,8 @@ public InfluxFunctionValue updateByIoTDBFunc( } String functionSqlSum = StringUtils.generateFunctionSql("sum", function.getParmaName(), path); - tsExecuteStatementResp = executeStatement(functionSqlSum, sessionid); + tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSqlSum, sessionid); list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { function.updateValueIoTDBFunc(null, influxFunctionValue); @@ -120,7 +85,8 @@ public InfluxFunctionValue updateByIoTDBFunc( { String functionSql = StringUtils.generateFunctionSql("sum", function.getParmaName(), path); - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -142,7 +108,8 @@ public InfluxFunctionValue updateByIoTDBFunc( StringUtils.generateFunctionSql("last_value", function.getParmaName(), path); functionName = "last_value"; } - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); Map map = QueryResultUtils.getColumnNameAndValue(tsExecuteStatementResp); for (String colume : map.keySet()) { Object o = map.get(colume); @@ -152,7 +119,8 @@ public InfluxFunctionValue updateByIoTDBFunc( String.format( "select %s from %s where %s=%s", function.getParmaName(), devicePath, fullPath, o); - TSExecuteStatementResp resp = executeStatement(specificSql, sessionid); + TSExecuteStatementResp resp = + NewInfluxDBServiceImpl.executeStatement(specificSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(resp); for (InfluxFunctionValue influxFunctionValue : list) { function.updateValueIoTDBFunc(influxFunctionValue); @@ -171,7 +139,8 @@ public InfluxFunctionValue updateByIoTDBFunc( functionSql = StringUtils.generateFunctionSql("min_value", function.getParmaName(), path); } - TSExecuteStatementResp tsExecuteStatementResp = executeStatement(functionSql, sessionid); + TSExecuteStatementResp tsExecuteStatementResp = + NewInfluxDBServiceImpl.executeStatement(functionSql, sessionid); List list = QueryResultUtils.getInfluxFunctionValues(tsExecuteStatementResp); for (InfluxFunctionValue influxFunctionValue : list) { @@ -185,15 +154,44 @@ public InfluxFunctionValue updateByIoTDBFunc( return function.calculateByIoTDBFunc(); } + /** + * If the function in the influxdb query request is also supported by IoTDB, use the IoTDB syntax + * to get the result + * + * @param database database of influxdb + * @param measurement measurement of influxdb + * @param function influxdb function + * @param sessionid session id + * @return influxdb function value + */ + @Override + public InfluxFunctionValue updateByIoTDBFunc( + String database, String measurement, InfluxFunction function, long sessionid) { + String path = "root." + database + "." + measurement; + return updateByIoTDBFunc(path, function, sessionid); + } + + /** + * Query the result according to the query SQL supported by IoTDB + * + * @param querySql query sql + * @param database database of influxdb + * @param measurement measurement of influxdb + * @param tagOrders tag orders + * @param fieldOrders field orders + * @param sessionId session id + * @return query result + */ @Override public QueryResult queryByConditions( String querySql, String database, String measurement, - ServiceProvider serviceProvider, + Map tagOrders, Map fieldOrders, long sessionId) { - TSExecuteStatementResp executeStatementResp = executeStatement(querySql, sessionId); + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.executeStatement(querySql, sessionId); return QueryResultUtils.iotdbResultConvertInfluxResult( executeStatementResp, database, measurement, fieldOrders); } diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java index b58b65f6bf5c..024fa011a617 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandler.java @@ -27,14 +27,13 @@ import org.apache.iotdb.db.protocol.influxdb.constant.InfluxSQLConstant; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunction; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; -import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManager; import org.apache.iotdb.db.protocol.influxdb.util.FieldUtils; import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; -import org.apache.iotdb.db.qp.physical.PhysicalPlan; import org.apache.iotdb.db.qp.physical.crud.QueryPlan; import org.apache.iotdb.db.query.context.QueryContext; import org.apache.iotdb.db.query.control.SessionManager; +import org.apache.iotdb.db.service.IoTDB; import org.apache.iotdb.db.service.basic.ServiceProvider; import org.apache.iotdb.tsfile.exception.filter.QueryFilterOptimizationException; import org.apache.iotdb.tsfile.read.common.Field; @@ -49,62 +48,28 @@ import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; +/** query handler for IoTDB */ public class QueryHandler extends AbstractQueryHandler { - @Override - public Map getFieldOrders( - String database, String measurement, ServiceProvider serviceProvider, long sessionID) { - Map fieldOrders = new HashMap<>(); - long queryId = ServiceProvider.SESSION_MANAGER.requestQueryId(true); - try { - String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; - PhysicalPlan physicalPlan = - serviceProvider.getPlanner().parseSQLToPhysicalPlan(showTimeseriesSql); - QueryContext queryContext = - serviceProvider.genQueryContext( - queryId, - true, - System.currentTimeMillis(), - showTimeseriesSql, - InfluxConstant.DEFAULT_CONNECTION_TIMEOUT_MS); - QueryDataSet queryDataSet = - serviceProvider.createQueryDataSet( - queryContext, physicalPlan, InfluxConstant.DEFAULT_FETCH_SIZE); - int fieldNums = 0; - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); - int tagOrderNums = tagOrders.size(); - while (queryDataSet.hasNext()) { - List fields = queryDataSet.next().getFields(); - String filed = StringUtils.getFieldByPath(fields.get(0).getStringValue()); - if (!fieldOrders.containsKey(filed)) { - // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, - // and finally all fields) - fieldOrders.put(filed, tagOrderNums + fieldNums + 1); - fieldNums++; - } - } - } catch (QueryProcessException - | TException - | StorageEngineException - | SQLException - | IOException - | InterruptedException - | QueryFilterOptimizationException - | MetadataException e) { - throw new InfluxDBException(e.getMessage()); - } finally { - ServiceProvider.SESSION_MANAGER.releaseQueryResourceNoExceptions(queryId); - } - return fieldOrders; - } + ServiceProvider serviceProvider = IoTDB.serviceProvider; + /** + * If the function in the influxdb query request is also supported by IoTDB, use the IoTDB syntax + * to get the result + * + * @param database database of influxdb + * @param measurement measurement of influxdb + * @param function influxdb function + * @param sessionid session id + * @return influxdb function value + */ @Override public InfluxFunctionValue updateByIoTDBFunc( - InfluxFunction function, ServiceProvider serviceProvider, String path, long sessionid) { + String database, String measurement, InfluxFunction function, long sessionid) { + String path = "root." + database + "." + measurement; switch (function.getFunctionName()) { case InfluxSQLConstant.COUNT: { @@ -476,12 +441,24 @@ public InfluxFunctionValue updateByIoTDBFunc( return function.calculateByIoTDBFunc(); } + /** + * Query the result according to the query SQL supported by IoTDB + * + * @param querySql query sql + * @param database database of influxdb + * @param measurement measurement of influxdb + * @param tagOrders tag orders + * @param fieldOrders field orders + * @param sessionId session id + * @return query result + * @throws AuthException + */ @Override public QueryResult queryByConditions( String querySql, String database, String measurement, - ServiceProvider serviceProvider, + Map tagOrders, Map fieldOrders, long sessionId) throws AuthException { diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java new file mode 100644 index 000000000000..7a6bf9bb88a3 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/QueryHandlerFactory.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.handler; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.service.thrift.impl.ClientRPCServiceImpl; + +/** Generate the corresponding QueryHandler object according to the configuration */ +public class QueryHandlerFactory { + + /** + * get QueryHandler object according to the configuration + * + * @return QueryHandler object + */ + public static AbstractQueryHandler getInstance() { + if (IoTDBDescriptor.getInstance() + .getConfig() + .getRpcImplClassName() + .equals(ClientRPCServiceImpl.class.getName())) { + if ("Tag".equals(IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + return new TagQueryHandler(); + } + return new NewQueryHandler(); + } else { + return new QueryHandler(); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java new file mode 100644 index 000000000000..76a76f23219a --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/handler/TagQueryHandler.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.handler; + +import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunction; +import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; +import org.apache.iotdb.db.protocol.influxdb.util.FilterUtils; +import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; +import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; +import org.apache.iotdb.tsfile.read.expression.IExpression; +import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression; + +import org.influxdb.dto.QueryResult; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** Query Handler for NewIoTDB When schema region is tag schema region */ +public class TagQueryHandler extends NewQueryHandler { + + /** + * If the function in the influxdb query request is also supported by IoTDB, use the IoTDB syntax + * to get the result + * + * @param database database of influxdb + * @param measurement measurement of influxdb + * @param function influxdb function + * @param sessionid session id + * @return influxdb function value + */ + @Override + public InfluxFunctionValue updateByIoTDBFunc( + String database, String measurement, InfluxFunction function, long sessionid) { + String path = "root." + database + ".measurement." + measurement; + return updateByIoTDBFunc(path, function, sessionid); + } + + /** + * Query the result according to the query SQL supported by IoTDB + * + * @param querySql query sql + * @param database database of influxdb + * @param measurement measurement of influxdb + * @param tagOrders tag orders + * @param fieldOrders field orders + * @param sessionId session id + * @return query result + */ + @Override + public QueryResult queryByConditions( + String querySql, + String database, + String measurement, + Map tagOrders, + Map fieldOrders, + long sessionId) { + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.executeStatement(querySql, sessionId); + return QueryResultUtils.iotdbResultConvertInfluxResult( + executeStatementResp, database, measurement, tagOrders, fieldOrders); + } + + @Override + public QueryResult queryByConditions( + List expressions, + String database, + String measurement, + Map fieldOrders, + Long sessionId) { + List fieldExpressions = new ArrayList<>(); + List tagExpressions = new ArrayList<>(); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionId); + for (IExpression expression : expressions) { + SingleSeriesExpression singleSeriesExpression = ((SingleSeriesExpression) expression); + // the current condition is in tag + if (tagOrders.containsKey(singleSeriesExpression.getSeriesPath().getFullPath())) { + tagExpressions.add(singleSeriesExpression); + } else { + fieldExpressions.add(singleSeriesExpression); + } + } + // construct the actual query path + StringBuilder curQueryPath = + new StringBuilder("root." + database + ".measurement." + measurement); + for (SingleSeriesExpression singleSeriesExpression : tagExpressions) { + String tagKey = singleSeriesExpression.getSeriesPath().getFullPath(); + String tagValue = + StringUtils.removeQuotation( + FilterUtils.getFilterStringValue(singleSeriesExpression.getFilter())); + curQueryPath.append(".").append(tagKey).append(".").append(tagValue); + } + curQueryPath.append(".**"); + + // construct actual query condition + StringBuilder realIotDBCondition = new StringBuilder(); + for (int i = 0; i < fieldExpressions.size(); i++) { + SingleSeriesExpression singleSeriesExpression = fieldExpressions.get(i); + if (i != 0) { + realIotDBCondition.append(" and "); + } + realIotDBCondition + .append(singleSeriesExpression.getSeriesPath().getFullPath()) + .append(" ") + .append((FilterUtils.getFilerSymbol(singleSeriesExpression.getFilter()))) + .append(" ") + .append(FilterUtils.getFilterStringValue(singleSeriesExpression.getFilter())); + } + // actual query SQL statement + String realQuerySql; + + realQuerySql = "select * from " + curQueryPath; + if (!(realIotDBCondition.length() == 0)) { + realQuerySql += " where " + realIotDBCondition; + } + realQuerySql += " align by device"; + return queryByConditions( + realQuerySql, database, measurement, tagOrders, fieldOrders, sessionId); + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java index 513b06e59f7b..2cb1740d4c1c 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/AbstractInfluxDBMetaManager.java @@ -22,8 +22,10 @@ import java.util.HashMap; import java.util.Map; +import java.util.Set; -public abstract class AbstractInfluxDBMetaManager { +/** InfluxDBMetaManager used in schema region is memory or schema file */ +public abstract class AbstractInfluxDBMetaManager implements IInfluxDBMetaManager { protected static final String SELECT_TAG_INFO_SQL = "select database_name,measurement_name,tag_name,tag_order from root.TAG_INFO "; @@ -32,7 +34,16 @@ public abstract class AbstractInfluxDBMetaManager { protected static Map>> database2Measurement2TagOrders = new HashMap<>(); - public static Map getTagOrders(String database, String measurement) { + /** + * get tag orders + * + * @param database database of influxdb + * @param measurement measurement of influxdb + * @param sessionID session id + * @return a map of tag orders + */ + @Override + public Map getTagOrders(String database, String measurement, long sessionID) { Map tagOrders = new HashMap<>(); Map> measurement2TagOrders = database2Measurement2TagOrders.get(database); @@ -45,10 +56,20 @@ public static Map getTagOrders(String database, String measurem return tagOrders; } - abstract void recover(); - + /** + * set storage group + * + * @param database database of influxdb + * @param sessionID session id + */ abstract void setStorageGroup(String database, long sessionID); + /** + * update tag info + * + * @param tagInfoRecords tagInfoRecords + * @param sessionID session id + */ abstract void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID); public final synchronized Map> createDatabase( @@ -69,8 +90,23 @@ public final synchronized Map getTagOrdersWithAutoCreatingSchem return createDatabase(database, sessionID).computeIfAbsent(measurement, m -> new HashMap<>()); } + /** + * generate time series path for insertion + * + * @param database database of influxdb + * @param measurement measurement of influxdb + * @param tags influxdb tags + * @param fields influxdb fields + * @param sessionID session id + * @return series path + */ + @Override public final synchronized String generatePath( - String database, String measurement, Map tags, long sessionID) { + String database, + String measurement, + Map tags, + Set fields, + long sessionID) { Map tagKeyToLayerOrders = getTagOrdersWithAutoCreatingSchema(database, measurement, sessionID); // to support rollback if fails to persisting new tag info diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java new file mode 100644 index 000000000000..3ba419b9abf4 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/IInfluxDBMetaManager.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.meta; + +import java.util.Map; +import java.util.Set; + +/** used to manage influxdb metadata */ +public interface IInfluxDBMetaManager { + + /** recover the influxdb metadata */ + void recover(); + + /** + * get field orders + * + * @param database database of influxdb + * @param measurement measurement of influxdb + * @param sessionId session id + * @return a map of field orders + */ + Map getFieldOrders(String database, String measurement, long sessionId); + + /** + * generate time series path for insertion + * + * @param database database of influxdb + * @param measurement measurement of influxdb + * @param tags influxdb tags + * @param fields influxdb fields + * @param sessionID session id + * @return series path + */ + String generatePath( + String database, + String measurement, + Map tags, + Set fields, + long sessionID); + + /** + * get tag orders + * + * @param database database of influxdb + * @param measurement measurement of influxdb + * @param sessionID session id + * @return a map of tag orders + */ + Map getTagOrders(String database, String measurement, long sessionID); +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java index f2e58de977ed..8465920353c9 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManager.java @@ -25,7 +25,10 @@ import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; import org.apache.iotdb.db.qp.Planner; +import org.apache.iotdb.db.qp.physical.PhysicalPlan; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; import org.apache.iotdb.db.qp.physical.crud.QueryPlan; import org.apache.iotdb.db.qp.physical.sys.SetStorageGroupPlan; @@ -46,6 +49,7 @@ import java.util.List; import java.util.Map; +/** InfluxDBMetaManager for IoTDB When schema region is memory or schema file */ public class InfluxDBMetaManager extends AbstractInfluxDBMetaManager { protected final Planner planner; @@ -114,6 +118,12 @@ public void recover() { } } + /** + * set storage group + * + * @param database database of influxdb + * @param sessionID session id + */ @Override public void setStorageGroup(String database, long sessionID) { try { @@ -130,6 +140,12 @@ public void setStorageGroup(String database, long sessionID) { } } + /** + * update tag info + * + * @param tagInfoRecords tagInfoRecords + * @param sessionID session id + */ @Override public void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID) { List plans = tagInfoRecords.convertToInsertRowPlans(); @@ -142,6 +158,61 @@ public void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID) } } + /** + * get field orders + * + * @param database database of influxdb + * @param measurement measurement of influxdb + * @param sessionID session id + * @return a map of field orders + */ + @Override + public Map getFieldOrders(String database, String measurement, long sessionID) { + Map fieldOrders = new HashMap<>(); + long queryId = ServiceProvider.SESSION_MANAGER.requestQueryId(true); + try { + String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; + PhysicalPlan physicalPlan = + serviceProvider.getPlanner().parseSQLToPhysicalPlan(showTimeseriesSql); + QueryContext queryContext = + serviceProvider.genQueryContext( + queryId, + true, + System.currentTimeMillis(), + showTimeseriesSql, + InfluxConstant.DEFAULT_CONNECTION_TIMEOUT_MS); + QueryDataSet queryDataSet = + serviceProvider.createQueryDataSet( + queryContext, physicalPlan, InfluxConstant.DEFAULT_FETCH_SIZE); + int fieldNums = 0; + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionID); + int tagOrderNums = tagOrders.size(); + while (queryDataSet.hasNext()) { + List fields = queryDataSet.next().getFields(); + String filed = StringUtils.getFieldByPath(fields.get(0).getStringValue()); + if (!fieldOrders.containsKey(filed)) { + // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, + // and finally all fields) + fieldOrders.put(filed, tagOrderNums + fieldNums + 1); + fieldNums++; + } + } + } catch (QueryProcessException + | TException + | StorageEngineException + | SQLException + | IOException + | InterruptedException + | QueryFilterOptimizationException + | MetadataException e) { + throw new InfluxDBException(e.getMessage()); + } finally { + ServiceProvider.SESSION_MANAGER.releaseQueryResourceNoExceptions(queryId); + } + return fieldOrders; + } + private static class InfluxDBMetaManagerHolder { private static final InfluxDBMetaManager INSTANCE = new InfluxDBMetaManager(); diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java new file mode 100644 index 000000000000..de2f6c993b24 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/InfluxDBMetaManagerFactory.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.meta; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.service.thrift.impl.ClientRPCServiceImpl; + +/** generate InfluxDBMetaManager object according to configuration */ +public class InfluxDBMetaManagerFactory { + + /** + * generate InfluxDBMetaManager object according to configuration + * + * @return InfluxDBMetaManager object + */ + public static IInfluxDBMetaManager getInstance() { + if (IoTDBDescriptor.getInstance() + .getConfig() + .getRpcImplClassName() + .equals(ClientRPCServiceImpl.class.getName())) { + if ("Tag".equals(IoTDBDescriptor.getInstance().getConfig().getSchemaEngineMode())) { + return TagInfluxDBMetaManager.getInstance(); + } + return NewInfluxDBMetaManager.getInstance(); + } else { + return InfluxDBMetaManager.getInstance(); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java index 5269a2bf443b..2cb6a694e571 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/NewInfluxDBMetaManager.java @@ -19,8 +19,8 @@ package org.apache.iotdb.db.protocol.influxdb.meta; import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.db.protocol.influxdb.handler.NewQueryHandler; import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; import org.apache.iotdb.db.service.thrift.impl.ClientRPCServiceImpl; import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; import org.apache.iotdb.rpc.IoTDBConnectionException; @@ -39,8 +39,10 @@ import java.util.List; import java.util.Map; +/** InfluxDBMetaManager for NewIoTDB When schema region is memory or schema file */ public class NewInfluxDBMetaManager extends AbstractInfluxDBMetaManager { + // NewIoTDB uses ClientRPCServiceImpl to handle the request private final ClientRPCServiceImpl clientRPCService; private NewInfluxDBMetaManager() { @@ -51,16 +53,20 @@ public static NewInfluxDBMetaManager getInstance() { return InfluxDBMetaManagerHolder.INSTANCE; } + /** recover the influxdb metadata */ @Override public void recover() { long sessionID = 0; try { TSOpenSessionResp tsOpenSessionResp = clientRPCService.openSession( - new TSOpenSessionReq().setUsername("root").setPassword("root")); + new TSOpenSessionReq() + .setUsername("root") + .setPassword("root") + .setZoneId("Asia/Shanghai")); sessionID = tsOpenSessionResp.getSessionId(); TSExecuteStatementResp resp = - NewQueryHandler.executeStatement(SELECT_TAG_INFO_SQL, sessionID); + NewInfluxDBServiceImpl.executeStatement(SELECT_TAG_INFO_SQL, sessionID); IoTDBJDBCDataSet dataSet = QueryResultUtils.creatIoTJDBCDataset(resp); try { Map> measurement2TagOrders; @@ -96,6 +102,12 @@ public void recover() { } } + /** + * set storage group + * + * @param database database of influxdb + * @param sessionID session id + */ @Override public void setStorageGroup(String database, long sessionID) { TSStatus status = clientRPCService.setStorageGroup(sessionID, "root." + database); @@ -106,6 +118,12 @@ public void setStorageGroup(String database, long sessionID) { throw new InfluxDBException(status.getMessage()); } + /** + * update tag info + * + * @param tagInfoRecords tagInfoRecords + * @param sessionID session id + */ @Override public void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID) { try { @@ -121,6 +139,37 @@ public void updateTagInfoRecords(TagInfoRecords tagInfoRecords, long sessionID) } } + /** + * get field orders + * + * @param database database of influxdb + * @param measurement measurement of influxdb + * @param sessionID session id + * @return a map of field orders + */ + @Override + public Map getFieldOrders(String database, String measurement, long sessionID) { + Map fieldOrders = new HashMap<>(); + String showTimeseriesSql = "show timeseries root." + database + '.' + measurement + ".**"; + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.executeStatement(showTimeseriesSql, sessionID); + List paths = QueryResultUtils.getFullPaths(executeStatementResp); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, sessionID); + int tagOrderNums = tagOrders.size(); + int fieldNums = 0; + for (String path : paths) { + String filed = StringUtils.getFieldByPath(path); + if (!fieldOrders.containsKey(filed)) { + // The corresponding order of fields is 1 + tagNum (the first is timestamp, then all tags, + // and finally all fields) + fieldOrders.put(filed, tagOrderNums + fieldNums + 1); + fieldNums++; + } + } + return fieldOrders; + } + private static class InfluxDBMetaManagerHolder { private static final NewInfluxDBMetaManager INSTANCE = new NewInfluxDBMetaManager(); diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java new file mode 100644 index 000000000000..29ee49356727 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/meta/TagInfluxDBMetaManager.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.protocol.influxdb.meta; + +import org.apache.iotdb.db.protocol.influxdb.util.QueryResultUtils; +import org.apache.iotdb.db.protocol.influxdb.util.StringUtils; +import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +/** InfluxDBMetaManager for NewIoTDB When schema region is tag schema region */ +public class TagInfluxDBMetaManager implements IInfluxDBMetaManager { + + // The storage group used to save influxdb metadata in IoTDB + private static final String STORAGE_GROUP_PATH = "root.influxdbmeta"; + + private static final String TAGS_SET = "set.tags"; + + private static final String FIELDS_SET = "set.fields"; + + private TagInfluxDBMetaManager() {} + + public static TagInfluxDBMetaManager getInstance() { + return TagInfluxDBMetaManagerHolder.INSTANCE; + } + + /** use tag schema region to save state information, no need to recover here */ + @Override + public void recover() {} + + /** + * get the fields information of influxdb corresponding database and measurement through tag + * schema region + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param sessionId session id + * @return field information + */ + @Override + public Map getFieldOrders(String database, String measurement, long sessionId) { + return getTimeseriesFieldOrders(database, measurement, FIELDS_SET, sessionId); + } + + /** + * convert the database,measurement,and tags of influxdb to device path of IoTDB,and save the tags + * and fields information of the database and measurement to the tag schema region + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param tags influxdb tags + * @param fields influxdb fields + * @param sessionID session id + * @return device path + */ + @Override + public String generatePath( + String database, + String measurement, + Map tags, + Set fields, + long sessionID) { + createInfluxDBMetaTimeseries(database, measurement, tags, fields, sessionID); + return generateDevicesPath(database, measurement, tags); + } + + private void createInfluxDBMetaTimeseries( + String database, + String measurement, + Map tags, + Set fields, + long sessionID) { + List fieldsList = new ArrayList<>(tags.keySet()); + createInfluxDBMetaTimeseries(database, measurement, TAGS_SET, fieldsList, sessionID); + fieldsList.clear(); + fieldsList.addAll(fields); + createInfluxDBMetaTimeseries(database, measurement, FIELDS_SET, fieldsList, sessionID); + } + + private void createInfluxDBMetaTimeseries( + String database, String measurement, String device, List fields, long sessionID) { + String statement = generateTimeseriesStatement(database, measurement, device, fields); + NewInfluxDBServiceImpl.executeStatement(statement, sessionID); + } + + private String generateTimeseriesStatement( + String database, String measurement, String device, List fields) { + StringBuilder timeseriesStatement = + new StringBuilder( + "create aligned timeseries " + + STORAGE_GROUP_PATH + + ".database." + + database + + ".measurement." + + measurement + + "." + + device + + "("); + for (int i = 0; i < fields.size() - 1; i++) { + String field = fields.get(i); + timeseriesStatement.append(field).append(" BOOLEAN, "); + } + timeseriesStatement.append(fields.get(fields.size() - 1)).append(" BOOLEAN)"); + return timeseriesStatement.toString(); + } + + /** + * get the tags information of influxdb corresponding database and measurement through tag schema + * region + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param sessionID session id + * @return tags information + */ + @Override + public Map getTagOrders(String database, String measurement, long sessionID) { + return getTimeseriesFieldOrders(database, measurement, TAGS_SET, sessionID); + } + + private Map getTimeseriesFieldOrders( + String database, String measurement, String device, long sessionID) { + TSExecuteStatementResp statementResp = + NewInfluxDBServiceImpl.executeStatement( + "show timeseries " + + STORAGE_GROUP_PATH + + ".database." + + database + + ".measurement." + + measurement + + "." + + device, + sessionID); + List timeseriesPaths = QueryResultUtils.getFullPaths(statementResp); + Map fieldOrders = new HashMap<>(); + for (String timeseriesPath : timeseriesPaths) { + String field = StringUtils.getFieldByPath(timeseriesPath); + fieldOrders.put(field, fieldOrders.size()); + } + return fieldOrders; + } + + /** + * convert the database,measurement,and tags of influxdb to device path of IoTDB,ensure that + * influxdb records with the same semantics generate the same device path, so the device path is + * generated in order after sorting the tags + * + * @param database influxdb database + * @param measurement influxdb measurement + * @param tags influxdb tags + * @return device path + */ + private String generateDevicesPath( + String database, String measurement, Map tags) { + TreeMap tagsMap = new TreeMap<>(tags); + tagsMap.put("measurement", measurement); + StringBuilder devicePath = new StringBuilder("root." + database); + for (String tagKey : tagsMap.keySet()) { + devicePath.append(".").append(tagKey).append(".").append(tagsMap.get(tagKey)); + } + return devicePath.toString(); + } + + private static class TagInfluxDBMetaManagerHolder { + private static final TagInfluxDBMetaManager INSTANCE = new TagInfluxDBMetaManager(); + + private TagInfluxDBMetaManagerHolder() {} + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java index 325971df2598..25199f6429f3 100644 --- a/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/protocol/influxdb/util/QueryResultUtils.java @@ -20,7 +20,7 @@ import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; import org.apache.iotdb.db.protocol.influxdb.function.InfluxFunctionValue; -import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; import org.apache.iotdb.db.query.dataset.AlignByDeviceDataSet; import org.apache.iotdb.rpc.IoTDBJDBCDataSet; import org.apache.iotdb.rpc.StatementExecutionException; @@ -82,7 +82,8 @@ public static QueryResult iotdbResultConvertInfluxResult( QueryResult.Series series = new QueryResult.Series(); series.setName(measurement); // gets the reverse map of the tag - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, -1); Map tagOrderReversed = tagOrders.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); @@ -294,6 +295,12 @@ public static boolean checkQueryResultNull(QueryResult queryResult) { return queryResult.getResults().get(0).getSeries() == null; } + /** + * parse time series paths from query results + * + * @param tsExecuteStatementResp query results + * @return time series paths + */ public static List getFullPaths(TSExecuteStatementResp tsExecuteStatementResp) { List res = new ArrayList<>(); IoTDBJDBCDataSet ioTDBJDBCDataSet = creatIoTJDBCDataset(tsExecuteStatementResp); @@ -309,6 +316,13 @@ public static List getFullPaths(TSExecuteStatementResp tsExecuteStatemen return res; } + /** + * Convert align by device query result of NewIoTDB to the query result of influxdb,used for + * Memory and schema_file schema region + * + * @param tsExecuteStatementResp NewIoTDB execute statement resp to be converted + * @return query results in influxdb format + */ public static QueryResult iotdbResultConvertInfluxResult( TSExecuteStatementResp tsExecuteStatementResp, String database, @@ -321,7 +335,8 @@ public static QueryResult iotdbResultConvertInfluxResult( QueryResult.Series series = new QueryResult.Series(); series.setName(measurement); // gets the reverse map of the tag - Map tagOrders = InfluxDBMetaManager.getTagOrders(database, measurement); + Map tagOrders = + InfluxDBMetaManagerFactory.getInstance().getTagOrders(database, measurement, -1); Map tagOrderReversed = tagOrders.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); @@ -381,6 +396,87 @@ public static QueryResult iotdbResultConvertInfluxResult( return queryResult; } + /** + * Convert align by device query result of NewIoTDB to the query result of influxdb,used for tag + * schema region + * + * @param tsExecuteStatementResp NewIoTDB execute statement resp to be converted + * @return query results in influxdb format + */ + public static QueryResult iotdbResultConvertInfluxResult( + TSExecuteStatementResp tsExecuteStatementResp, + String database, + String measurement, + Map tagOrders, + Map fieldOrders) { + if (tsExecuteStatementResp == null) { + return getNullQueryResult(); + } + // generate series + QueryResult.Series series = new QueryResult.Series(); + series.setName(measurement); + Map tagOrderReversed = + tagOrders.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + int tagSize = tagOrderReversed.size(); + Map fieldOrdersReversed = + fieldOrders.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + ArrayList tagList = new ArrayList<>(); + for (int i = 0; i < tagSize; i++) { + tagList.add(tagOrderReversed.get(i)); + } + + ArrayList fieldList = new ArrayList<>(); + for (int i = 0; i < fieldOrders.size(); i++) { + fieldList.add(fieldOrdersReversed.get(i)); + } + + ArrayList columns = new ArrayList<>(); + columns.add("time"); + columns.addAll(tagList); + columns.addAll(fieldList); + // insert columns into series + series.setColumns(columns); + List> values = new ArrayList<>(); + IoTDBJDBCDataSet ioTDBJDBCDataSet = creatIoTJDBCDataset(tsExecuteStatementResp); + try { + while (ioTDBJDBCDataSet.hasCachedResults()) { + Object[] value = new Object[columns.size()]; + ioTDBJDBCDataSet.constructOneRow(); + value[0] = Long.valueOf(ioTDBJDBCDataSet.getValueByName("Time")); + String deviceName = ioTDBJDBCDataSet.getValueByName("Device"); + String[] deviceNameList = deviceName.split("\\."); + for (int i = 2; i < deviceNameList.length; i += 2) { + if (tagOrders.containsKey(deviceNameList[i])) { + int position = tagOrders.get(deviceNameList[i]) + 1; + value[position] = deviceNameList[i + 1]; + } + } + for (int i = 3; i <= ioTDBJDBCDataSet.columnNameList.size(); i++) { + Object o = ioTDBJDBCDataSet.getObject(ioTDBJDBCDataSet.findColumnNameByIndex(i)); + if (o != null) { + // insert the value of filed into it + int position = fieldOrders.get(ioTDBJDBCDataSet.findColumnNameByIndex(i)) + tagSize + 1; + value[position] = o; + } + } + values.add(Arrays.asList(value)); + } + } catch (Exception e) { + e.printStackTrace(); + } + + series.setValues(values); + + QueryResult queryResult = new QueryResult(); + QueryResult.Result result = new QueryResult.Result(); + result.setSeries(new ArrayList<>(Arrays.asList(series))); + queryResult.setResults(new ArrayList<>(Arrays.asList(result))); + + return queryResult; + } + public static List getInfluxFunctionValues( TSExecuteStatementResp tsExecuteStatementResp) { IoTDBJDBCDataSet ioTDBJDBCDataSet = creatIoTJDBCDataset(tsExecuteStatementResp); diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java index 89e5429dd19a..8209b434876a 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/InfluxDBServiceImpl.java @@ -151,8 +151,7 @@ public InfluxTSStatus createDatabase(InfluxCreateDatabaseReq req) { public InfluxQueryResultRsp query(InfluxQueryReq req) throws TException { Operator operator = InfluxDBLogicalGenerator.generate(req.command); queryHandler.checkInfluxDBQueryOperator(operator); - return queryHandler.queryInfluxDB( - req.database, (InfluxQueryOperator) operator, req.sessionId, IoTDB.serviceProvider); + return queryHandler.queryInfluxDB(req.database, (InfluxQueryOperator) operator, req.sessionId); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java index 422bc27fd46a..38a3a4d157c3 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java +++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/NewInfluxDBServiceImpl.java @@ -19,17 +19,17 @@ package org.apache.iotdb.db.service.thrift.impl; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.db.protocol.influxdb.constant.InfluxConstant; import org.apache.iotdb.db.protocol.influxdb.dto.IoTDBPoint; import org.apache.iotdb.db.protocol.influxdb.handler.AbstractQueryHandler; -import org.apache.iotdb.db.protocol.influxdb.handler.NewQueryHandler; +import org.apache.iotdb.db.protocol.influxdb.handler.QueryHandlerFactory; import org.apache.iotdb.db.protocol.influxdb.input.InfluxLineParser; -import org.apache.iotdb.db.protocol.influxdb.meta.AbstractInfluxDBMetaManager; -import org.apache.iotdb.db.protocol.influxdb.meta.NewInfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.IInfluxDBMetaManager; +import org.apache.iotdb.db.protocol.influxdb.meta.InfluxDBMetaManagerFactory; import org.apache.iotdb.db.protocol.influxdb.operator.InfluxQueryOperator; import org.apache.iotdb.db.protocol.influxdb.sql.InfluxDBLogicalGenerator; import org.apache.iotdb.db.protocol.influxdb.util.InfluxReqAndRespUtils; import org.apache.iotdb.db.qp.logical.Operator; -import org.apache.iotdb.db.service.IoTDB; import org.apache.iotdb.db.utils.DataTypeUtils; import org.apache.iotdb.protocol.influxdb.rpc.thrift.InfluxCloseSessionReq; import org.apache.iotdb.protocol.influxdb.rpc.thrift.InfluxCreateDatabaseReq; @@ -42,6 +42,8 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.iotdb.service.rpc.thrift.TSCloseSessionReq; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementReq; +import org.apache.iotdb.service.rpc.thrift.TSExecuteStatementResp; import org.apache.iotdb.service.rpc.thrift.TSInsertRecordReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionReq; import org.apache.iotdb.service.rpc.thrift.TSOpenSessionResp; @@ -53,23 +55,34 @@ import java.util.ArrayList; import java.util.List; +/** + * When using NewIoTDB, use this object to handle read and write requests of the influxdb protocol + */ public class NewInfluxDBServiceImpl implements IInfluxDBServiceWithHandler { private static final ClientRPCServiceImpl clientRPCService = new ClientRPCServiceImpl(); - private final AbstractInfluxDBMetaManager metaManager; + private final IInfluxDBMetaManager metaManager; private final AbstractQueryHandler queryHandler; public NewInfluxDBServiceImpl() { - metaManager = NewInfluxDBMetaManager.getInstance(); - queryHandler = new NewQueryHandler(); + metaManager = InfluxDBMetaManagerFactory.getInstance(); + metaManager.recover(); + queryHandler = QueryHandlerFactory.getInstance(); } public static ClientRPCServiceImpl getClientRPCService() { return clientRPCService; } + /** + * open session + * + * @param req InfluxOpenSessionReq + * @return InfluxOpenSessionResp + * @throws TException + */ @Override public InfluxOpenSessionResp openSession(InfluxOpenSessionReq req) throws TException { TSOpenSessionReq tsOpenSessionReq = InfluxReqAndRespUtils.convertOpenSessionReq(req); @@ -77,6 +90,12 @@ public InfluxOpenSessionResp openSession(InfluxOpenSessionReq req) throws TExcep return InfluxReqAndRespUtils.convertOpenSessionResp(tsOpenSessionResp); } + /** + * close session + * + * @param req InfluxCloseSessionReq + * @return InfluxTSStatus + */ @Override public InfluxTSStatus closeSession(InfluxCloseSessionReq req) { TSCloseSessionReq tsCloseSessionReq = InfluxReqAndRespUtils.convertCloseSessionReq(req); @@ -84,6 +103,12 @@ public InfluxTSStatus closeSession(InfluxCloseSessionReq req) { return DataTypeUtils.RPCStatusToInfluxDBTSStatus(tsStatus); } + /** + * Handling insert requests + * + * @param req InfluxWritePointsReq + * @return InfluxTSStatus + */ @Override public InfluxTSStatus writePoints(InfluxWritePointsReq req) { List tsStatusList = new ArrayList<>(); @@ -102,6 +127,12 @@ public InfluxTSStatus writePoints(InfluxWritePointsReq req) { return new InfluxTSStatus().setCode(executeCode).setSubStatus(tsStatusList); } + /** + * Create a database in the influxdb semantics + * + * @param req InfluxCreateDatabaseReq + * @return InfluxTSStatus + */ @Override public InfluxTSStatus createDatabase(InfluxCreateDatabaseReq req) { TSStatus tsStatus = @@ -113,14 +144,40 @@ public InfluxTSStatus createDatabase(InfluxCreateDatabaseReq req) { return DataTypeUtils.RPCStatusToInfluxDBTSStatus(tsStatus); } + /** + * Process query requests + * + * @param req InfluxQueryReq + * @return InfluxQueryResultRsp + * @throws TException + */ @Override public InfluxQueryResultRsp query(InfluxQueryReq req) throws TException { Operator operator = InfluxDBLogicalGenerator.generate(req.command); queryHandler.checkInfluxDBQueryOperator(operator); - return queryHandler.queryInfluxDB( - req.database, (InfluxQueryOperator) operator, req.sessionId, IoTDB.serviceProvider); + return queryHandler.queryInfluxDB(req.database, (InfluxQueryOperator) operator, req.sessionId); + } + + /** + * execute sql statement + * + * @param sql sql statement + * @param sessionId session id + * @return TSExecuteStatementResp + */ + public static TSExecuteStatementResp executeStatement(String sql, long sessionId) { + TSExecuteStatementReq tsExecuteStatementReq = new TSExecuteStatementReq(); + tsExecuteStatementReq.setStatement(sql); + tsExecuteStatementReq.setSessionId(sessionId); + tsExecuteStatementReq.setStatementId( + NewInfluxDBServiceImpl.getClientRPCService().requestStatementId(sessionId)); + tsExecuteStatementReq.setFetchSize(InfluxConstant.DEFAULT_FETCH_SIZE); + TSExecuteStatementResp executeStatementResp = + NewInfluxDBServiceImpl.getClientRPCService().executeStatement(tsExecuteStatementReq); + return executeStatementResp; } + /** handle client exit, close session and resource */ @Override public void handleClientExit() { clientRPCService.handleClientExit();