From 0a0483f7089c1dee9150afe749189744414bc412 Mon Sep 17 00:00:00 2001 From: W1y1r <2730956796@qq.com> Date: Fri, 15 Nov 2024 10:52:22 +0800 Subject: [PATCH] Dataode actively listens and loads TsFile --- src/.vuepress/sidebar/V1.3.x/en.ts | 1 + src/.vuepress/sidebar/V1.3.x/zh.ts | 1 + src/.vuepress/sidebar_timecho/V1.3.x/en.ts | 1 + src/.vuepress/sidebar_timecho/V1.3.x/zh.ts | 1 + .../Master/Tools-System/Data-Import-Tool.md | 190 ++++++++++++++++++ .../latest/Tools-System/Data-Import-Tool.md | 190 ++++++++++++++++++ .../Master/Tools-System/Data-Import-Tool.md | 182 +++++++++++++++++ .../latest/Tools-System/Data-Import-Tool.md | 182 +++++++++++++++++ 8 files changed, 748 insertions(+) create mode 100644 src/UserGuide/Master/Tools-System/Data-Import-Tool.md create mode 100644 src/UserGuide/latest/Tools-System/Data-Import-Tool.md create mode 100644 src/zh/UserGuide/Master/Tools-System/Data-Import-Tool.md create mode 100644 src/zh/UserGuide/latest/Tools-System/Data-Import-Tool.md diff --git a/src/.vuepress/sidebar/V1.3.x/en.ts b/src/.vuepress/sidebar/V1.3.x/en.ts index daa3fb6f7..974140ed6 100644 --- a/src/.vuepress/sidebar/V1.3.x/en.ts +++ b/src/.vuepress/sidebar/V1.3.x/en.ts @@ -101,6 +101,7 @@ export const enSidebar = { { text: 'Maintenance Tool', link: 'Maintenance-Tool_apache' }, { text: 'Data Import Export Script', link: 'Data-Import-Export-Tool' }, { text: 'TsFile Import Export Script', link: 'TsFile-Import-Export-Tool' }, + { text: 'Data Import Tools', link: 'Data-Import-Tool' }, ], }, { diff --git a/src/.vuepress/sidebar/V1.3.x/zh.ts b/src/.vuepress/sidebar/V1.3.x/zh.ts index 4fa047b3c..ebb92c5e9 100644 --- a/src/.vuepress/sidebar/V1.3.x/zh.ts +++ b/src/.vuepress/sidebar/V1.3.x/zh.ts @@ -101,6 +101,7 @@ export const zhSidebar = { { text: '运维工具', link: 'Maintenance-Tool_apache' }, { text: '数据导入导出脚本', link: 'Data-Import-Export-Tool' }, { text: 'TsFile导入导出脚本', link: 'TsFile-Import-Export-Tool' }, + { text: '导入工具', link: 'Data-Import-Tool' }, ], }, { diff --git a/src/.vuepress/sidebar_timecho/V1.3.x/en.ts b/src/.vuepress/sidebar_timecho/V1.3.x/en.ts index 063b1daff..23c6b76e3 100644 --- a/src/.vuepress/sidebar_timecho/V1.3.x/en.ts +++ b/src/.vuepress/sidebar_timecho/V1.3.x/en.ts @@ -110,6 +110,7 @@ export const enSidebar = { { text: 'Maintenance Tool', link: 'Maintenance-Tool_timecho' }, { text: 'Data Import Export Script', link: 'Data-Import-Export-Tool' }, { text: 'TsFile Import Export Script', link: 'TsFile-Import-Export-Tool' }, + { text: 'Data Import Tools', link: 'Data-Import-Tool' }, ], }, { diff --git a/src/.vuepress/sidebar_timecho/V1.3.x/zh.ts b/src/.vuepress/sidebar_timecho/V1.3.x/zh.ts index 3e8f86497..b03f9d91c 100644 --- a/src/.vuepress/sidebar_timecho/V1.3.x/zh.ts +++ b/src/.vuepress/sidebar_timecho/V1.3.x/zh.ts @@ -110,6 +110,7 @@ export const zhSidebar = { { text: '运维工具', link: 'Maintenance-Tool_timecho' }, { text: '数据导入导出脚本', link: 'Data-Import-Export-Tool' }, { text: 'TsFile导入导出脚本', link: 'TsFile-Import-Export-Tool' }, + { text: '导入工具', link: 'Data-Import-Tool' }, ], }, { diff --git a/src/UserGuide/Master/Tools-System/Data-Import-Tool.md b/src/UserGuide/Master/Tools-System/Data-Import-Tool.md new file mode 100644 index 000000000..68d9d4769 --- /dev/null +++ b/src/UserGuide/Master/Tools-System/Data-Import-Tool.md @@ -0,0 +1,190 @@ +# Data Import Tools + +## 1. Introduction to Import Tools + +The IoTDB import tool suite is divided into two types: active listening and loading TsFile and import scripts. The import scripts include the `import-tsfile.sh/bat` script specifically for importing TsFile files and the `import-data.sh/bat` script that supports the import of CSV and SQL formats, which can be used for single file import or batch import of a directory. + + +## 2. Active Listening and Loading TsFile + +Users only need to specify the sending end file directory and the receiving end target directory through the data collection forwarding program, and the files will be automatically synchronized to the scheduled receiving end directory. With the active listening feature of DataNode, files placed in the scheduled receiving end directory can be automatically detected and loaded by IoTDB without any additional loading operations. This automated process not only simplifies the user's operational steps but also reduces potential errors in the operation process, effectively reducing the complexity for users during use. + + +### 2.1 Configuration Parameters + +| **Configuration Parameter** | **Description** | **Value Range** | **Required** | **Default Value** | **Loading Method** | +| -------------------------------------------- | ------------------------------------------------------------ | -------------------------- | ------------ | ---------------------- | ---------------- | +| load_active_listening_enable | Whether to enable the DataNode's active listening and loading of tsfile functionality (default is enabled). | Boolean: true，false | Optional | true | Hot Loading | +| load_active_listening_dirs | The directories to be listened to (automatically includes subdirectories of the directory), if there are multiple, separate with “,”. The default directory is ext/load/pending (supports hot loading). | String: one or more file directories | Optional | ext/load/pending | Hot Loading | +| load_active_listening_fail_dir | The directory to which files are transferred after the execution of loading tsfile files fails, only one directory can be configured. | String: one file directory | Optional | ext/load/failed | Hot Loading | +| load_active_listening_max_thread_num | The maximum number of threads to perform loading tsfile tasks simultaneously. The default value when the parameter is commented out is max(1, CPU core count / 2). When the user sets a value not in the range [1, CPU core count / 2], it will be set to the default value (1, CPU core count / 2). | Long: [1, Long.MAX_VALUE] | Optional | max(1, CPU core count / 2) | Effective after restart | +| load_active_listening_check_interval_seconds | Active listening polling interval in seconds. The function of actively listening to tsfile is achieved by polling the folder. This configuration specifies the time interval between two checks of load_active_listening_dirs, and the next check will be executed after load_active_listening_check_interval_seconds seconds of each check. When the user sets the polling interval to less than 1, it will be set to the default value of 5 seconds. | Long: [1, Long.MAX_VALUE] | Optional | 5 | Effective after restart | + + +### 2.2 Precautions + +1. If there is a mods file in the files to be loaded, the mods file should be moved to the listening directory first, and then the tsfile files should be moved, with the mods file and the corresponding tsfile file in the same directory. This prevents the loading of tsfile files without the corresponding mods files. + +```SQL +FUNCTION moveFilesToListeningDirectory(sourceDirectory, listeningDirectory) + // Move mods files + modsFiles = searchFiles(sourceDirectory, "*mods*") + IF modsFiles IS NOT EMPTY + FOR EACH file IN modsFiles + MOVE(file, listeningDirectory) + END FOR + END IF + + // Move tsfile files + tsfileFiles = searchFiles(sourceDirectory, "*tsfile*") + IF tsfileFiles IS NOT EMPTY + FOR EACH file IN tsfileFiles + MOVE(file, listeningDirectory) + END FOR + END IF +END FUNCTION + +FUNCTION searchFiles(directory, pattern) + matchedFiles = [] + FOR EACH file IN directory.files + IF file.name MATCHES pattern + APPEND file TO matchedFiles + END IF + END FOR + RETURN matchedFiles +END FUNCTION + +FUNCTION MOVE(sourceFile, targetDirectory) + // Implement the logic of moving files from sourceFile to targetDirectory +END FUNCTION +``` + +2. Prohibit setting the receiver directory of Pipe, the data directory for storing data, etc., as the listening directory. + +3. Prohibit `load_active_listening_fail_dir` from having the same directory as `load_active_listening_dirs`, or each other's nesting. + +4. Ensure that the `load_active_listening_dirs` directory has sufficient permissions. After the load is successful, the files will be deleted. If there is no delete permission, it will lead to repeated loading. + +## 3. Import Scripts + +### 3.1 Supported Data Types + +- CSV: Pure text format for storing formatted data, must comply with the specified CSV format below. + +- SQL: Files containing IoTDB SQL statements. + +- TsFile: IoTDB native time series data file format. + + +### 3.2 load-tsfile Script + +Supports TsFile: The file format for time series used in IoTDB. + +#### 3.2.1 Command + +```Bash +# Unix/OS X +>tools/load-tsfile.sh -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] + +# Windows +>tools\load-tsfile.bat -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] +``` + +#### 3.2.2 Parameter Introduction + + +| **Parameter** | **Description** | **Required** | **Default** | +| -------- | ------------------------------------------------------------ | ----------------------------------- | ------------------- | +| -h | Hostname | No | root | +| -p | Port | No | root | +| -u | Username | No | 127.0.0.1 | +| -pw | Password | No | 6667 | +| -s | The local directory path of the script file (folder) to be loaded | Yes | | +| -os | none: Do not delete
mv: Move successful files to the target folder
cp: Hard link (copy) successful files to the target folder
delete: Delete | Yes | | +| -sd | When --on_success is mv or cp, the target folder for mv or cp. The file name of the file becomes the folder flattened and then concatenated with the original file name. | When --on_success is mv or cp, it is required to fill in Yes | ${EXEC_DIR}/success | +| -of | none: Skip
mv: Move failed files to the target folder
cp: Hard link (copy) failed files to the target folder
delete: Delete | Yes | | +| -fd | When --on_fail is specified as mv or cp, the target folder for mv or cp. The file name of the file becomes the folder flattened and then concatenated with the original file name. | When --on_fail is specified as mv or cp, it is required to fill in | ${EXEC_DIR}/fail | +| -tn | Maximum number of parallel threads | Yes | 8 | + +#### 3.2.3 Running Examples + +```Bash +# Unix/OS X +> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 +> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 + +# Windows +> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 +> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 +``` + +## 3.3 import-data Script + +Supports CSV: A plain text format for storing formatted data, which needs to be constructed according to the specified CSV format below. + +Supports SQL: A file containing custom SQL statements. + +#### 3.3.1 Command + +```Bash +# Unix/OS X +>tools/import-data.sh -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] + +# Windows +>tools\import-data.bat -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] +``` + +#### 3.3.2 Parameter Introduction + +| **Parameter** | **Definition** | **Required** | **Default** | +| --------- | ------------------------------------------------------------ | ------------ | ------------------------ | +| -h | Hostname | No | 127.0.0.1 | +| -p | Port | No | 6667 | +| -u | Username | No | root | +| -pw | Password | No | root | +| -s | Specify the data to be imported, here you can specify files or folders. If a folder is specified, all files with suffixes of csv or sql in the folder will be batch imported (In V1.3.2, the parameter is `-f`) | Yes | | +| -fd | Specify the directory for storing failed SQL files. If this parameter is not specified, failed files will be saved in the source data directory. Note: For unsupported SQL, illegal SQL, and failed SQL, they will be put into the failed directory under the failed file (default is the file name with `.failed` suffix) | No |The source filename with `.failed` suffix | +| -aligned | Specify whether to use the `aligned` interface, options are true or false. Note: This parameter is only effective when importing csv files. | No | false | +| -batch | Used to specify the number of data points per batch (minimum value is 1, maximum value is Integer.*MAX_VALUE*). If the program reports the error `org.apache.thrift.transport.TTransportException: Frame size larger than protect max size`, you can appropriately reduce this parameter. | No | 100000 | +| -tp | Specify the time precision, options include `ms` (milliseconds), `ns` (nanoseconds), `us` (microseconds) | No | ms | +| -lpf | Specify the number of data lines written per failed file (In V1.3.2, the parameter is `-linesPerFailedFile`) | No | 10000 | +| -typeInfer | Used to specify type inference rules, such as . Note: Used to specify type inference rules. `srcTsDataType` includes `boolean`, `int`, `long`, `float`, `double`, `NaN`. `dstTsDataType` includes `boolean`, `int`, `long`, `float`, `double`, `text`. When `srcTsDataType` is `boolean`, `dstTsDataType` can only be `boolean` or `text`. When `srcTsDataType` is `NaN`, `dstTsDataType` can only be `float`, `double`, or `text`. When `srcTsDataType` is a numerical type, the precision of `dstTsDataType` needs to be higher than `srcTsDataType`. For example: `-typeInfer boolean=text,float=double` | No | | + +#### 3.3.3 Running Examples + +- Import the `dump0_0.sql` data in the current `data` directory to the local IoTDB database. + +```Bash +# Unix/OS X +>tools/import-data.sh -s ./data/dump0_0.sql +# Windows +>tools/import-data.bat -s ./data/dump0_0.sql +``` + +- Import all data in the current `data` directory in an aligned manner to the local IoTDB database. + +```Bash +# Unix/OS X +>tools/import-data.sh -s ./data/ -fd ./failed/ -aligned true +# Windows +>tools/import-data.bat -s ./data/ -fd ./failed/ -aligned true +``` + +- Import the `dump0_0.csv` data in the current `data` directory to the local IoTDB database. + +```Bash +# Unix/OS X +>tools/import-data.sh -s ./data/dump0_0.csv -fd ./failed/ +# Windows +>tools/import-data.bat -s ./data/dump0_0.csv -fd ./failed/ +``` + +- Import the `dump0_0.csv` data in the current `data` directory in an aligned manner, batch import 100000 lines to the IoTDB database on the host with IP `192.168.100.1`, record failures in the current `failed` directory, with a maximum of 1000 lines per file. + + +```Bash +# Unix/OS X +>tools/import-data.sh -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +# Windows +>tools/import-data.bat -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +``` \ No newline at end of file diff --git a/src/UserGuide/latest/Tools-System/Data-Import-Tool.md b/src/UserGuide/latest/Tools-System/Data-Import-Tool.md new file mode 100644 index 000000000..68d9d4769 --- /dev/null +++ b/src/UserGuide/latest/Tools-System/Data-Import-Tool.md @@ -0,0 +1,190 @@ +# Data Import Tools + +## 1. Introduction to Import Tools + +The IoTDB import tool suite is divided into two types: active listening and loading TsFile and import scripts. The import scripts include the `import-tsfile.sh/bat` script specifically for importing TsFile files and the `import-data.sh/bat` script that supports the import of CSV and SQL formats, which can be used for single file import or batch import of a directory. + + +## 2. Active Listening and Loading TsFile + +Users only need to specify the sending end file directory and the receiving end target directory through the data collection forwarding program, and the files will be automatically synchronized to the scheduled receiving end directory. With the active listening feature of DataNode, files placed in the scheduled receiving end directory can be automatically detected and loaded by IoTDB without any additional loading operations. This automated process not only simplifies the user's operational steps but also reduces potential errors in the operation process, effectively reducing the complexity for users during use. + + +### 2.1 Configuration Parameters + +| **Configuration Parameter** | **Description** | **Value Range** | **Required** | **Default Value** | **Loading Method** | +| -------------------------------------------- | ------------------------------------------------------------ | -------------------------- | ------------ | ---------------------- | ---------------- | +| load_active_listening_enable | Whether to enable the DataNode's active listening and loading of tsfile functionality (default is enabled). | Boolean: true，false | Optional | true | Hot Loading | +| load_active_listening_dirs | The directories to be listened to (automatically includes subdirectories of the directory), if there are multiple, separate with “,”. The default directory is ext/load/pending (supports hot loading). | String: one or more file directories | Optional | ext/load/pending | Hot Loading | +| load_active_listening_fail_dir | The directory to which files are transferred after the execution of loading tsfile files fails, only one directory can be configured. | String: one file directory | Optional | ext/load/failed | Hot Loading | +| load_active_listening_max_thread_num | The maximum number of threads to perform loading tsfile tasks simultaneously. The default value when the parameter is commented out is max(1, CPU core count / 2). When the user sets a value not in the range [1, CPU core count / 2], it will be set to the default value (1, CPU core count / 2). | Long: [1, Long.MAX_VALUE] | Optional | max(1, CPU core count / 2) | Effective after restart | +| load_active_listening_check_interval_seconds | Active listening polling interval in seconds. The function of actively listening to tsfile is achieved by polling the folder. This configuration specifies the time interval between two checks of load_active_listening_dirs, and the next check will be executed after load_active_listening_check_interval_seconds seconds of each check. When the user sets the polling interval to less than 1, it will be set to the default value of 5 seconds. | Long: [1, Long.MAX_VALUE] | Optional | 5 | Effective after restart | + + +### 2.2 Precautions + +1. If there is a mods file in the files to be loaded, the mods file should be moved to the listening directory first, and then the tsfile files should be moved, with the mods file and the corresponding tsfile file in the same directory. This prevents the loading of tsfile files without the corresponding mods files. + +```SQL +FUNCTION moveFilesToListeningDirectory(sourceDirectory, listeningDirectory) + // Move mods files + modsFiles = searchFiles(sourceDirectory, "*mods*") + IF modsFiles IS NOT EMPTY + FOR EACH file IN modsFiles + MOVE(file, listeningDirectory) + END FOR + END IF + + // Move tsfile files + tsfileFiles = searchFiles(sourceDirectory, "*tsfile*") + IF tsfileFiles IS NOT EMPTY + FOR EACH file IN tsfileFiles + MOVE(file, listeningDirectory) + END FOR + END IF +END FUNCTION + +FUNCTION searchFiles(directory, pattern) + matchedFiles = [] + FOR EACH file IN directory.files + IF file.name MATCHES pattern + APPEND file TO matchedFiles + END IF + END FOR + RETURN matchedFiles +END FUNCTION + +FUNCTION MOVE(sourceFile, targetDirectory) + // Implement the logic of moving files from sourceFile to targetDirectory +END FUNCTION +``` + +2. Prohibit setting the receiver directory of Pipe, the data directory for storing data, etc., as the listening directory. + +3. Prohibit `load_active_listening_fail_dir` from having the same directory as `load_active_listening_dirs`, or each other's nesting. + +4. Ensure that the `load_active_listening_dirs` directory has sufficient permissions. After the load is successful, the files will be deleted. If there is no delete permission, it will lead to repeated loading. + +## 3. Import Scripts + +### 3.1 Supported Data Types + +- CSV: Pure text format for storing formatted data, must comply with the specified CSV format below. + +- SQL: Files containing IoTDB SQL statements. + +- TsFile: IoTDB native time series data file format. + + +### 3.2 load-tsfile Script + +Supports TsFile: The file format for time series used in IoTDB. + +#### 3.2.1 Command + +```Bash +# Unix/OS X +>tools/load-tsfile.sh -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] + +# Windows +>tools\load-tsfile.bat -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] +``` + +#### 3.2.2 Parameter Introduction + + +| **Parameter** | **Description** | **Required** | **Default** | +| -------- | ------------------------------------------------------------ | ----------------------------------- | ------------------- | +| -h | Hostname | No | root | +| -p | Port | No | root | +| -u | Username | No | 127.0.0.1 | +| -pw | Password | No | 6667 | +| -s | The local directory path of the script file (folder) to be loaded | Yes | | +| -os | none: Do not delete
mv: Move successful files to the target folder
cp: Hard link (copy) successful files to the target folder
delete: Delete | Yes | | +| -sd | When --on_success is mv or cp, the target folder for mv or cp. The file name of the file becomes the folder flattened and then concatenated with the original file name. | When --on_success is mv or cp, it is required to fill in Yes | ${EXEC_DIR}/success | +| -of | none: Skip
mv: Move failed files to the target folder
cp: Hard link (copy) failed files to the target folder
delete: Delete | Yes | | +| -fd | When --on_fail is specified as mv or cp, the target folder for mv or cp. The file name of the file becomes the folder flattened and then concatenated with the original file name. | When --on_fail is specified as mv or cp, it is required to fill in | ${EXEC_DIR}/fail | +| -tn | Maximum number of parallel threads | Yes | 8 | + +#### 3.2.3 Running Examples + +```Bash +# Unix/OS X +> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 +> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 + +# Windows +> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 +> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 +``` + +## 3.3 import-data Script + +Supports CSV: A plain text format for storing formatted data, which needs to be constructed according to the specified CSV format below. + +Supports SQL: A file containing custom SQL statements. + +#### 3.3.1 Command + +```Bash +# Unix/OS X +>tools/import-data.sh -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] + +# Windows +>tools\import-data.bat -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] +``` + +#### 3.3.2 Parameter Introduction + +| **Parameter** | **Definition** | **Required** | **Default** | +| --------- | ------------------------------------------------------------ | ------------ | ------------------------ | +| -h | Hostname | No | 127.0.0.1 | +| -p | Port | No | 6667 | +| -u | Username | No | root | +| -pw | Password | No | root | +| -s | Specify the data to be imported, here you can specify files or folders. If a folder is specified, all files with suffixes of csv or sql in the folder will be batch imported (In V1.3.2, the parameter is `-f`) | Yes | | +| -fd | Specify the directory for storing failed SQL files. If this parameter is not specified, failed files will be saved in the source data directory. Note: For unsupported SQL, illegal SQL, and failed SQL, they will be put into the failed directory under the failed file (default is the file name with `.failed` suffix) | No |The source filename with `.failed` suffix | +| -aligned | Specify whether to use the `aligned` interface, options are true or false. Note: This parameter is only effective when importing csv files. | No | false | +| -batch | Used to specify the number of data points per batch (minimum value is 1, maximum value is Integer.*MAX_VALUE*). If the program reports the error `org.apache.thrift.transport.TTransportException: Frame size larger than protect max size`, you can appropriately reduce this parameter. | No | 100000 | +| -tp | Specify the time precision, options include `ms` (milliseconds), `ns` (nanoseconds), `us` (microseconds) | No | ms | +| -lpf | Specify the number of data lines written per failed file (In V1.3.2, the parameter is `-linesPerFailedFile`) | No | 10000 | +| -typeInfer | Used to specify type inference rules, such as . Note: Used to specify type inference rules. `srcTsDataType` includes `boolean`, `int`, `long`, `float`, `double`, `NaN`. `dstTsDataType` includes `boolean`, `int`, `long`, `float`, `double`, `text`. When `srcTsDataType` is `boolean`, `dstTsDataType` can only be `boolean` or `text`. When `srcTsDataType` is `NaN`, `dstTsDataType` can only be `float`, `double`, or `text`. When `srcTsDataType` is a numerical type, the precision of `dstTsDataType` needs to be higher than `srcTsDataType`. For example: `-typeInfer boolean=text,float=double` | No | | + +#### 3.3.3 Running Examples + +- Import the `dump0_0.sql` data in the current `data` directory to the local IoTDB database. + +```Bash +# Unix/OS X +>tools/import-data.sh -s ./data/dump0_0.sql +# Windows +>tools/import-data.bat -s ./data/dump0_0.sql +``` + +- Import all data in the current `data` directory in an aligned manner to the local IoTDB database. + +```Bash +# Unix/OS X +>tools/import-data.sh -s ./data/ -fd ./failed/ -aligned true +# Windows +>tools/import-data.bat -s ./data/ -fd ./failed/ -aligned true +``` + +- Import the `dump0_0.csv` data in the current `data` directory to the local IoTDB database. + +```Bash +# Unix/OS X +>tools/import-data.sh -s ./data/dump0_0.csv -fd ./failed/ +# Windows +>tools/import-data.bat -s ./data/dump0_0.csv -fd ./failed/ +``` + +- Import the `dump0_0.csv` data in the current `data` directory in an aligned manner, batch import 100000 lines to the IoTDB database on the host with IP `192.168.100.1`, record failures in the current `failed` directory, with a maximum of 1000 lines per file. + + +```Bash +# Unix/OS X +>tools/import-data.sh -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +# Windows +>tools/import-data.bat -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +``` \ No newline at end of file diff --git a/src/zh/UserGuide/Master/Tools-System/Data-Import-Tool.md b/src/zh/UserGuide/Master/Tools-System/Data-Import-Tool.md new file mode 100644 index 000000000..f1c4e6c84 --- /dev/null +++ b/src/zh/UserGuide/Master/Tools-System/Data-Import-Tool.md @@ -0,0 +1,182 @@ +# 数据导入工具 + +## 1. 导入工具介绍 + +IoTDB 导入工具包分为两种：主动监听并加载 TsFile 和导入脚本。导入脚本中含专门用于导入 TsFile 文件的`import-tsfile.sh/bat` 脚本和支持 CSV 和 SQL 格式的导入的 `import-data.sh/bat` 脚本，可用于单个文件导入或一个目录的批量导入。 + +## 2. 主动监听并加载 TsFile + +用户只需通过数据采集转发程序指定发送端的文件目录和接收端的目标目录，文件的自动同步至预定接收端目录。通过DataNode的主动监听功能，放入预定接收端目录中的文件，IoTDB便能自动检测并加载这些文件，无需手动执行任何额外的加载操作。这种自动化流程不仅简化了用户的操作步骤，还减少了操作过程中可能出现的错误，有效降低了用户在使用过程中的复杂性。 + +### 2.1 配置参数 + +| **配置参数** | **参数说明** | **value 取值范围** | **是否必填** | **默认值** | **加载方式** | +| -------------------------------------------- | ------------------------------------------------------------ | -------------------------- | ------------ | ---------------------- | ---------------- | +| load_active_listening_enable | 是否开启 DataNode 主动监听并且加载 tsfile 的功能（默认开启）。 | Boolean: true，false | 选填 | true | 热加载 | +| load_active_listening_dirs | 需要监听的目录（自动包括目录中的子目录），如有多个使用 “，“ 隔开默认的目录为 ext/load/pending（支持热装载） | String: 一个或多个文件目录 | 选填 | ext/load/pending | 热加载 | +| load_active_listening_fail_dir | 执行加载 tsfile 文件失败后将文件转存的目录，只能配置一个 | String: 一个文件目录 | 选填 | ext/load/failed | 热加载 | +| load_active_listening_max_thread_num | 同时执行加载 tsfile 任务的最大线程数，参数被注释掉时的默值为 max(1, CPU 核心数 / 2)，当用户设置的值不在这个区间[1, CPU核心数 /2]内时，会设置为默认值 (1, CPU 核心数 / 2) | Long: [1, Long.MAX_VALUE] | 选填 | max(1, CPU 核心数 / 2) | 重启后生效 | +| load_active_listening_check_interval_seconds | 主动监听轮询间隔，单位秒。主动监听 tsfile 的功能是通过轮询检查文件夹实现的。该配置指定了两次检查 load_active_listening_dirs 的时间间隔，每次检查完成 load_active_listening_check_interval_seconds 秒后，会执行下一次检查。当用户设置的轮询间隔小于 1 时，会被设置为默认值 5 秒 | Long: [1, Long.MAX_VALUE] | 选填 | 5 | 重启后生效 | + + +### 2.2 注意事项 + +1. 如果待加载的文件中，存在 mods 文件，应优先将 mods 文件移动到监听目录下面，然后再移动 tsfile 文件，且 mods 文件应和对应的 tsfile 文件处于同一目录。防止加载到 tsfile 文件时，加载不到对应的 mods 文件 +```SQL +FUNCTION moveFilesToListeningDirectory(sourceDirectory, listeningDirectory) + // 移动 mods 文件 + modsFiles = searchFiles(sourceDirectory, "*mods*") + IF modsFiles IS NOT EMPTY + FOR EACH file IN modsFiles + MOVE(file, listeningDirectory) + END FOR + END IF + + // 移动 tsfile 文件 + tsfileFiles = searchFiles(sourceDirectory, "*tsfile*") + IF tsfileFiles IS NOT EMPTY + FOR EACH file IN tsfileFiles + MOVE(file, listeningDirectory) + END FOR + END IF +END FUNCTION + +FUNCTION searchFiles(directory, pattern) + matchedFiles = [] + FOR EACH file IN directory.files + IF file.name MATCHES pattern + APPEND file TO matchedFiles + END IF + END FOR + RETURN matchedFiles +END FUNCTION + +FUNCTION MOVE(sourceFile, targetDirectory) + // 实现文件从 sourceFile 移动到 targetDirectory 的逻辑 +END FUNCTION +``` + +2. 禁止设置 Pipe 的 receiver 目录、存放数据的 data 目录等作为监听目录 + +3. 禁止 `load_active_listening_fail_dir` 与 `load_active_listening_dirs` 存在相同的目录，或者互相嵌套 + +4. 保证 `load_active_listening_dirs` 目录有足够的权限，在加载成功之后，文件将会被删除，如果没有删除权限，则会重复加载 + +## 3.导入脚本 + +### 3.1 支持的数据类型 + +- CSV：纯文本格式，存储格式化数据，需符合下文指定 CSV 格式 +- SQL：包含 IoTDB SQL 语句的文件 +- TsFile： IoTDB 原生时序数据文件格式 + +### 3.2 load-tsfile 脚本 + +支持 TsFile： IoTDB 中使用的时间序列的文件格式 + +#### 3.2.1 运行命令 + +```Bash +# Unix/OS X +>tools/load-tsfile.sh -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] + +# Windows +>tools\load-tsfile.bat -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] +``` + +#### 3.2.2 参数介绍 + +| **参数** | **定义** | **是否必填** | **默认** | +| -------- | ------------------------------------------------------------ | ----------------------------------- | ------------------- | +| -h | 主机名 | 否 | root | +| -p | 端口号 | 否 | root | +| -u | 用户名 | 否 | 127.0.0.1 | +| -pw | 密码 | 否 | 6667 | +| -s | 待加载的脚本文件(夹)的本地目录路径 | 是 | | +| -os | none：不删除
mv：移动成功的文件到目标文件夹
cp：硬连接（拷贝）成功的文件到目标文件夹
delete：删除 | 是 | | +| -sd | 当--on_succcess为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当--on_succcess为mv或cp时需要填写是 | ${EXEC_DIR}/success | +| -of | none：跳过
mv：移动失败的文件到目标文件夹
cp：硬连接（拷贝）失败的文件到目标文件夹
delete：删除 | 是 | | +| -fd | 当--on_fail指定为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当--on_fail指定为mv或cp时需要填写 | ${EXEC_DIR}/fail | +| -tn | 最大并行线程数 | 是 | 8 | + +#### 3.2.3 运行示例： + +```Bash +# Unix/OS X +> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 +> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 + +# Windows +> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 +> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 +``` + +## 3.3 import-data 脚本 + +支持 CSV：纯文本格式，存储格式化数据，需按照下文指定 CSV 格式进行构造 + +支持 SQL：包含自定义 SQL 语句的文件 + +#### 3.3.1 运行命令： + +```Bash +# Unix/OS X +>tools/import-data.sh -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] + +# Windows +>tools\import-data.bat -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] +``` + +#### 3.3.2 参数介绍： + +| **参数** | **定义** | **是否必填** | **默认** | +| ---------- | ------------------------------------------------------------ | ------------ | ------------------------- | +| -h | 数据库IP地址 | 否 | 127.0.0.1 | +| -p | 数据库端口 | 否 | 6667 | +| -u | 数据库连接用户名 | 否 | root | +| -pw | 数据库连接密码 | 否 | root | +| -s | 指定想要导入的数据，这里可以指定文件或者文件夹。如果指定的是文件夹，将会把文件夹中所有的后缀为 csv 或者 sql 的文件进行批量导入(V1.3.2版本参数是`-f`) | 是 | | +| -fd | 指定存放失败 SQL 文件的目录，如果未指定这个参数，失败的文件将会被保存到源数据的目录中。 说明：对于不支持的 SQL ，不合法的 SQL ，执行失败的 SQL 都会放到失败目录下的失败文件里（默认为 文件名.failed） | 否 | 源文件名加上`.failed`后缀 | +| -aligned | 指定是否使用`aligned`接口，选项为 true 或者 false 说明：这个参数只在导入文件为csv文件时生效 | 否 | false | +| -batch | 用于指定每一批插入的数据的点数（最小值为1，最大值为 Integer.*MAX_VALUE*）。如果程序报了`org.apache.thrift.transport.TTransportException: Frame size larger than protect max size`这个错的话，就可以适当的调低这个参数。 | 否 | 100000 | +| -tp | 指定时间精度，可选值包括`ms`（毫秒），`ns`（纳秒），`us`（微秒） | 否 | ms | +| -lpf | 指定每个导入失败文件写入数据的行数(V1.3.2版本参数是`-linesPerFailedFile`) | 否 | 10000 | +| -typeInfer | 用于指定类型推断规则，如。 说明：用于指定类型推断规则.`srcTsDataType` 包括 `boolean`,`int`,`long`,`float`,`double`,`NaN`.`dstTsDataType` 包括 `boolean`,`int`,`long`,`float`,`double`,`text`.当`srcTsDataType`为`boolean`, `dstTsDataType`只能为`boolean`或`text`.当`srcTsDataType`为`NaN`, `dstTsDataType`只能为`float`, `double`或`text`.当`srcTsDataType`为数值类型, `dstTsDataType`的精度需要高于`srcTsDataType`.例如:`-typeInfer boolean=text,float=double` | 否 | | + +#### 3.3.3 运行示例： + +- 导入当前`data`目录下的`dump0_0.sql`数据到本机 IoTDB 数据库中。 + +```Bash +# Unix/OS X +>tools/import-data.sh -s ./data/dump0_0.sql +# Windows +>tools/import-data.bat -s ./data/dump0_0.sql +``` + +- 将当前`data`目录下的所有数据以对齐的方式导入到本机 IoTDB 数据库中。 + +```Bash +# Unix/OS X +>tools/import-data.sh -s ./data/ -fd ./failed/ -aligned true +# Windows +>tools/import-data.bat -s ./data/ -fd ./failed/ -aligned true +``` + +- 导入当前`data`目录下的`dump0_0.csv`数据到本机 IoTDB 数据库中。 + +```Bash +# Unix/OS X +>tools/import-data.sh -s ./data/dump0_0.csv -fd ./failed/ +# Windows +>tools/import-data.bat -s ./data/dump0_0.csv -fd ./failed/ +``` + +- 将当前`data`目录下的`dump0_0.csv`数据以对齐的方式，一批导入100000条导入到`192.168.100.1`IP所在主机的 IoTDB 数据库中，失败的记录记在当前`failed`目录下，每个文件最多记1000条。 + +```Bash +# Unix/OS X +>tools/import-data.sh -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +# Windows +>tools/import-data.bat -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +``` \ No newline at end of file diff --git a/src/zh/UserGuide/latest/Tools-System/Data-Import-Tool.md b/src/zh/UserGuide/latest/Tools-System/Data-Import-Tool.md new file mode 100644 index 000000000..f1c4e6c84 --- /dev/null +++ b/src/zh/UserGuide/latest/Tools-System/Data-Import-Tool.md @@ -0,0 +1,182 @@ +# 数据导入工具 + +## 1. 导入工具介绍 + +IoTDB 导入工具包分为两种：主动监听并加载 TsFile 和导入脚本。导入脚本中含专门用于导入 TsFile 文件的`import-tsfile.sh/bat` 脚本和支持 CSV 和 SQL 格式的导入的 `import-data.sh/bat` 脚本，可用于单个文件导入或一个目录的批量导入。 + +## 2. 主动监听并加载 TsFile + +用户只需通过数据采集转发程序指定发送端的文件目录和接收端的目标目录，文件的自动同步至预定接收端目录。通过DataNode的主动监听功能，放入预定接收端目录中的文件，IoTDB便能自动检测并加载这些文件，无需手动执行任何额外的加载操作。这种自动化流程不仅简化了用户的操作步骤，还减少了操作过程中可能出现的错误，有效降低了用户在使用过程中的复杂性。 + +### 2.1 配置参数 + +| **配置参数** | **参数说明** | **value 取值范围** | **是否必填** | **默认值** | **加载方式** | +| -------------------------------------------- | ------------------------------------------------------------ | -------------------------- | ------------ | ---------------------- | ---------------- | +| load_active_listening_enable | 是否开启 DataNode 主动监听并且加载 tsfile 的功能（默认开启）。 | Boolean: true，false | 选填 | true | 热加载 | +| load_active_listening_dirs | 需要监听的目录（自动包括目录中的子目录），如有多个使用 “，“ 隔开默认的目录为 ext/load/pending（支持热装载） | String: 一个或多个文件目录 | 选填 | ext/load/pending | 热加载 | +| load_active_listening_fail_dir | 执行加载 tsfile 文件失败后将文件转存的目录，只能配置一个 | String: 一个文件目录 | 选填 | ext/load/failed | 热加载 | +| load_active_listening_max_thread_num | 同时执行加载 tsfile 任务的最大线程数，参数被注释掉时的默值为 max(1, CPU 核心数 / 2)，当用户设置的值不在这个区间[1, CPU核心数 /2]内时，会设置为默认值 (1, CPU 核心数 / 2) | Long: [1, Long.MAX_VALUE] | 选填 | max(1, CPU 核心数 / 2) | 重启后生效 | +| load_active_listening_check_interval_seconds | 主动监听轮询间隔，单位秒。主动监听 tsfile 的功能是通过轮询检查文件夹实现的。该配置指定了两次检查 load_active_listening_dirs 的时间间隔，每次检查完成 load_active_listening_check_interval_seconds 秒后，会执行下一次检查。当用户设置的轮询间隔小于 1 时，会被设置为默认值 5 秒 | Long: [1, Long.MAX_VALUE] | 选填 | 5 | 重启后生效 | + + +### 2.2 注意事项 + +1. 如果待加载的文件中，存在 mods 文件，应优先将 mods 文件移动到监听目录下面，然后再移动 tsfile 文件，且 mods 文件应和对应的 tsfile 文件处于同一目录。防止加载到 tsfile 文件时，加载不到对应的 mods 文件 +```SQL +FUNCTION moveFilesToListeningDirectory(sourceDirectory, listeningDirectory) + // 移动 mods 文件 + modsFiles = searchFiles(sourceDirectory, "*mods*") + IF modsFiles IS NOT EMPTY + FOR EACH file IN modsFiles + MOVE(file, listeningDirectory) + END FOR + END IF + + // 移动 tsfile 文件 + tsfileFiles = searchFiles(sourceDirectory, "*tsfile*") + IF tsfileFiles IS NOT EMPTY + FOR EACH file IN tsfileFiles + MOVE(file, listeningDirectory) + END FOR + END IF +END FUNCTION + +FUNCTION searchFiles(directory, pattern) + matchedFiles = [] + FOR EACH file IN directory.files + IF file.name MATCHES pattern + APPEND file TO matchedFiles + END IF + END FOR + RETURN matchedFiles +END FUNCTION + +FUNCTION MOVE(sourceFile, targetDirectory) + // 实现文件从 sourceFile 移动到 targetDirectory 的逻辑 +END FUNCTION +``` + +2. 禁止设置 Pipe 的 receiver 目录、存放数据的 data 目录等作为监听目录 + +3. 禁止 `load_active_listening_fail_dir` 与 `load_active_listening_dirs` 存在相同的目录，或者互相嵌套 + +4. 保证 `load_active_listening_dirs` 目录有足够的权限，在加载成功之后，文件将会被删除，如果没有删除权限，则会重复加载 + +## 3.导入脚本 + +### 3.1 支持的数据类型 + +- CSV：纯文本格式，存储格式化数据，需符合下文指定 CSV 格式 +- SQL：包含 IoTDB SQL 语句的文件 +- TsFile： IoTDB 原生时序数据文件格式 + +### 3.2 load-tsfile 脚本 + +支持 TsFile： IoTDB 中使用的时间序列的文件格式 + +#### 3.2.1 运行命令 + +```Bash +# Unix/OS X +>tools/load-tsfile.sh -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] + +# Windows +>tools\load-tsfile.bat -h -p -u -pw -s -os [-sd ] -of [-fd ] [-tn ] +``` + +#### 3.2.2 参数介绍 + +| **参数** | **定义** | **是否必填** | **默认** | +| -------- | ------------------------------------------------------------ | ----------------------------------- | ------------------- | +| -h | 主机名 | 否 | root | +| -p | 端口号 | 否 | root | +| -u | 用户名 | 否 | 127.0.0.1 | +| -pw | 密码 | 否 | 6667 | +| -s | 待加载的脚本文件(夹)的本地目录路径 | 是 | | +| -os | none：不删除
mv：移动成功的文件到目标文件夹
cp：硬连接（拷贝）成功的文件到目标文件夹
delete：删除 | 是 | | +| -sd | 当--on_succcess为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当--on_succcess为mv或cp时需要填写是 | ${EXEC_DIR}/success | +| -of | none：跳过
mv：移动失败的文件到目标文件夹
cp：硬连接（拷贝）失败的文件到目标文件夹
delete：删除 | 是 | | +| -fd | 当--on_fail指定为mv或cp时，mv或cp的目标文件夹。文件的文件名变为文件夹打平后拼接原有文件名 | 当--on_fail指定为mv或cp时需要填写 | ${EXEC_DIR}/fail | +| -tn | 最大并行线程数 | 是 | 8 | + +#### 3.2.3 运行示例： + +```Bash +# Unix/OS X +> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 +> tools/load-tsfile.sh -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 + +# Windows +> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os mv -of cp -sd /path/success/dir -fd /path/failure/dir -tn 8 +> tools/load_data.bat -h 127.0.0.1 -p 6667 -u root -pw root -s /path/sql -os delete -of delete -tn 8 +``` + +## 3.3 import-data 脚本 + +支持 CSV：纯文本格式，存储格式化数据，需按照下文指定 CSV 格式进行构造 + +支持 SQL：包含自定义 SQL 语句的文件 + +#### 3.3.1 运行命令： + +```Bash +# Unix/OS X +>tools/import-data.sh -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] + +# Windows +>tools\import-data.bat -h -p -u -pw -s [-fd <./failedDirectory> -aligned -batch -tp -typeInfer -lpf ] +``` + +#### 3.3.2 参数介绍： + +| **参数** | **定义** | **是否必填** | **默认** | +| ---------- | ------------------------------------------------------------ | ------------ | ------------------------- | +| -h | 数据库IP地址 | 否 | 127.0.0.1 | +| -p | 数据库端口 | 否 | 6667 | +| -u | 数据库连接用户名 | 否 | root | +| -pw | 数据库连接密码 | 否 | root | +| -s | 指定想要导入的数据，这里可以指定文件或者文件夹。如果指定的是文件夹，将会把文件夹中所有的后缀为 csv 或者 sql 的文件进行批量导入(V1.3.2版本参数是`-f`) | 是 | | +| -fd | 指定存放失败 SQL 文件的目录，如果未指定这个参数，失败的文件将会被保存到源数据的目录中。 说明：对于不支持的 SQL ，不合法的 SQL ，执行失败的 SQL 都会放到失败目录下的失败文件里（默认为 文件名.failed） | 否 | 源文件名加上`.failed`后缀 | +| -aligned | 指定是否使用`aligned`接口，选项为 true 或者 false 说明：这个参数只在导入文件为csv文件时生效 | 否 | false | +| -batch | 用于指定每一批插入的数据的点数（最小值为1，最大值为 Integer.*MAX_VALUE*）。如果程序报了`org.apache.thrift.transport.TTransportException: Frame size larger than protect max size`这个错的话，就可以适当的调低这个参数。 | 否 | 100000 | +| -tp | 指定时间精度，可选值包括`ms`（毫秒），`ns`（纳秒），`us`（微秒） | 否 | ms | +| -lpf | 指定每个导入失败文件写入数据的行数(V1.3.2版本参数是`-linesPerFailedFile`) | 否 | 10000 | +| -typeInfer | 用于指定类型推断规则，如。 说明：用于指定类型推断规则.`srcTsDataType` 包括 `boolean`,`int`,`long`,`float`,`double`,`NaN`.`dstTsDataType` 包括 `boolean`,`int`,`long`,`float`,`double`,`text`.当`srcTsDataType`为`boolean`, `dstTsDataType`只能为`boolean`或`text`.当`srcTsDataType`为`NaN`, `dstTsDataType`只能为`float`, `double`或`text`.当`srcTsDataType`为数值类型, `dstTsDataType`的精度需要高于`srcTsDataType`.例如:`-typeInfer boolean=text,float=double` | 否 | | + +#### 3.3.3 运行示例： + +- 导入当前`data`目录下的`dump0_0.sql`数据到本机 IoTDB 数据库中。 + +```Bash +# Unix/OS X +>tools/import-data.sh -s ./data/dump0_0.sql +# Windows +>tools/import-data.bat -s ./data/dump0_0.sql +``` + +- 将当前`data`目录下的所有数据以对齐的方式导入到本机 IoTDB 数据库中。 + +```Bash +# Unix/OS X +>tools/import-data.sh -s ./data/ -fd ./failed/ -aligned true +# Windows +>tools/import-data.bat -s ./data/ -fd ./failed/ -aligned true +``` + +- 导入当前`data`目录下的`dump0_0.csv`数据到本机 IoTDB 数据库中。 + +```Bash +# Unix/OS X +>tools/import-data.sh -s ./data/dump0_0.csv -fd ./failed/ +# Windows +>tools/import-data.bat -s ./data/dump0_0.csv -fd ./failed/ +``` + +- 将当前`data`目录下的`dump0_0.csv`数据以对齐的方式，一批导入100000条导入到`192.168.100.1`IP所在主机的 IoTDB 数据库中，失败的记录记在当前`failed`目录下，每个文件最多记1000条。 + +```Bash +# Unix/OS X +>tools/import-data.sh -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +# Windows +>tools/import-data.bat -h 192.168.100.1 -p 6667 -u root -pw root -s ./data/dump0_0.csv -fd ./failed/ -aligned true -batch 100000 -tp ms -typeInfer boolean=text,float=double -lpf 1000 +``` \ No newline at end of file